From b52307c58a947732e6fd22e088c3a0cb02689839 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 19:21:13 +0000 Subject: [PATCH 1/7] gemm: name the base of the small-matrix kernel offsets No functional change. With DYNAMIC_ARCH the small-matrix kernel tables in interface/gemm*.c hold FUNC_OFFSET()s, which are offsets into gotoblas_t, and add them to gotoblas. Spell that base FUNC_BASE(func), next to FUNC_OFFSET(func), and reach it through GEMM_SMALL_KERNEL_BASE in the same way as the offsets themselves. This lets the kernels move out of gotoblas_t without interface/gemm*.c knowing where they went. Co-Authored-By: Claude Opus 5.5 --- common_c.h | 1 + common_d.h | 1 + common_macro.h | 5 +++++ common_param.h | 2 ++ common_s.h | 1 + common_sb.h | 1 + common_z.h | 1 + interface/gemm.c | 2 +- interface/gemm_batch.c | 2 +- interface/gemm_batch_strided.c | 2 +- 10 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common_c.h b/common_c.h index f12561127b..d0a4e595fb 100644 --- a/common_c.h +++ b/common_c.h @@ -501,6 +501,7 @@ #endif +#define CGEMM_SMALL_KERNEL_BASE FUNC_BASE(cgemm_small_kernel_nn) #define CGEMM_SMALL_KERNEL_NN FUNC_OFFSET(cgemm_small_kernel_nn) #define CGEMM_SMALL_KERNEL_NT FUNC_OFFSET(cgemm_small_kernel_nt) #define CGEMM_SMALL_KERNEL_NR FUNC_OFFSET(cgemm_small_kernel_nr) diff --git a/common_d.h b/common_d.h index 30fbe17b55..6eef671401 100644 --- a/common_d.h +++ b/common_d.h @@ -344,6 +344,7 @@ #endif +#define DGEMM_SMALL_KERNEL_BASE FUNC_BASE(dgemm_small_kernel_nn) #define DGEMM_SMALL_KERNEL_NN FUNC_OFFSET(dgemm_small_kernel_nn) #define DGEMM_SMALL_KERNEL_NT FUNC_OFFSET(dgemm_small_kernel_nt) #define DGEMM_SMALL_KERNEL_TN FUNC_OFFSET(dgemm_small_kernel_tn) diff --git a/common_macro.h b/common_macro.h index 8ed813742b..8b46a5ce9f 100644 --- a/common_macro.h +++ b/common_macro.h @@ -697,6 +697,7 @@ #define GEMM_SMALL_MATRIX_PERMIT DGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE DGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN DGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT DGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN DGEMM_SMALL_KERNEL_TN @@ -1103,6 +1104,7 @@ #define GEMM_SMALL_MATRIX_PERMIT SBGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE SBGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN SBGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT SBGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN SBGEMM_SMALL_KERNEL_TN @@ -1446,6 +1448,7 @@ #define GEMM_SMALL_MATRIX_PERMIT SGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE SGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN @@ -2393,6 +2396,7 @@ #define GEMM_SMALL_MATRIX_PERMIT ZGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE ZGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN ZGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT ZGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_NR ZGEMM_SMALL_KERNEL_NR @@ -2913,6 +2917,7 @@ #define GEMM_SMALL_MATRIX_PERMIT CGEMM_SMALL_MATRIX_PERMIT +#define GEMM_SMALL_KERNEL_BASE CGEMM_SMALL_KERNEL_BASE #define GEMM_SMALL_KERNEL_NN CGEMM_SMALL_KERNEL_NN #define GEMM_SMALL_KERNEL_NT CGEMM_SMALL_KERNEL_NT #define GEMM_SMALL_KERNEL_NR CGEMM_SMALL_KERNEL_NR diff --git a/common_param.h b/common_param.h index ce04ca6aa1..83f9ea3b13 100644 --- a/common_param.h +++ b/common_param.h @@ -1322,6 +1322,8 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL extern gotoblas_t *gotoblas; #define FUNC_OFFSET(func) (size_t)(&((gotoblas_t *)NULL)->func) +/* The address that FUNC_OFFSET(func) is an offset from. */ +#define FUNC_BASE(func) ((char *)gotoblas) #define DTB_ENTRIES gotoblas -> dtb_entries #define GEMM_OFFSET_A gotoblas -> offsetA diff --git a/common_s.h b/common_s.h index 54b59595f3..06b25b89fd 100644 --- a/common_s.h +++ b/common_s.h @@ -392,6 +392,7 @@ #endif +#define SGEMM_SMALL_KERNEL_BASE FUNC_BASE(sgemm_small_kernel_nn) #define SGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sgemm_small_kernel_nn) #define SGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sgemm_small_kernel_nt) #define SGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sgemm_small_kernel_tn) diff --git a/common_sb.h b/common_sb.h index d21e7a563b..d79c3e0cbc 100644 --- a/common_sb.h +++ b/common_sb.h @@ -45,6 +45,7 @@ #define SBGEMM_SMALL_MATRIX_PERMIT gotoblas -> sbgemm_small_matrix_permit #endif +#define SBGEMM_SMALL_KERNEL_BASE FUNC_BASE(sbgemm_small_kernel_nn) #define SBGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sbgemm_small_kernel_nn) #define SBGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sbgemm_small_kernel_nt) #define SBGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sbgemm_small_kernel_tn) diff --git a/common_z.h b/common_z.h index 24f88595df..abd13dc719 100644 --- a/common_z.h +++ b/common_z.h @@ -501,6 +501,7 @@ #endif +#define ZGEMM_SMALL_KERNEL_BASE FUNC_BASE(zgemm_small_kernel_nn) #define ZGEMM_SMALL_KERNEL_NN FUNC_OFFSET(zgemm_small_kernel_nn) #define ZGEMM_SMALL_KERNEL_NT FUNC_OFFSET(zgemm_small_kernel_nt) #define ZGEMM_SMALL_KERNEL_NR FUNC_OFFSET(zgemm_small_kernel_nr) diff --git a/interface/gemm.c b/interface/gemm.c index f1fd49cb15..8572b052bf 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -128,7 +128,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif diff --git a/interface/gemm_batch.c b/interface/gemm_batch.c index 7a8789e820..b0f2f5adc7 100644 --- a/interface/gemm_batch.c +++ b/interface/gemm_batch.c @@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif diff --git a/interface/gemm_batch_strided.c b/interface/gemm_batch_strided.c index 026eeb50e2..92d4326506 100644 --- a/interface/gemm_batch_strided.c +++ b/interface/gemm_batch_strided.c @@ -75,7 +75,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #ifndef DYNAMIC_ARCH #define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) #else -#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)(GEMM_SMALL_KERNEL_BASE + (size_t)(table[idx])))) #endif From 1524c8f037da99a75ac0818206aaaf92f1275a26 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:17:45 +0000 Subject: [PATCH 2/7] build: generate dyn_cores.h from DYNAMIC_CORE Preparation for numbering the DYNAMIC_ARCH cores; nothing includes the header yet. dyn_cores.h holds one X-macro, OPENBLAS_CORE_LIST(X, arg), which expands to X(CORE, arg) for each core of the build, in sorted order and without duplicates. Makefile.system and cmake/arch.cmake each write it where they finish computing DYNAMIC_CORE. dyn_cores.h is written by the top-level make only - sub-makes inherit OPENBLAS_DYN_CORES - so that parallel sub-makes never rewrite it under a compile. It is emitted through $(HASH) because GNU Make 3.81 reads a literal '#' inside $(shell ...) as a comment. cmake rewrites it only when the list changes, and puts it in the build directory, which the kernel targets now have on their include path. A stale copy left in the source tree by make would shadow cmake's, so cmake refuses to configure with one there, as it already does for config_kernel.h. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + Makefile | 2 +- Makefile.system | 9 +++++++++ cmake/arch.cmake | 17 +++++++++++++++++ kernel/CMakeLists.txt | 2 +- 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3d6ddb5052..2b9f4c869d 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ CMakeCache.txt CMakeFiles/* .vscode **/__pycache__ +dyn_cores.h diff --git a/Makefile b/Makefile index ae1da5e9fe..8a117f9927 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ clean :: ifeq ($(OSNAME), Darwin) @rm -rf getarch.dSYM getarch_2nd.dSYM endif - @rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h st* *.dylib + @rm -f Makefile.conf config.h Makefile_kernel.conf config_kernel.h dyn_cores.h st* *.dylib @rm -f cblas.tmp cblas.tmp2 @touch $(NETLIB_LAPACK_DIR)/make.inc @$(MAKE) -C $(NETLIB_LAPACK_DIR) clean diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..9c89611d7a 100644 --- a/Makefile.system +++ b/Makefile.system @@ -867,6 +867,15 @@ override DYNAMIC_ARCH= endif endif +# dyn_cores.h tells common_param.h which cores this DYNAMIC_ARCH build has. +ifeq ($(DYNAMIC_ARCH), 1) +ifndef OPENBLAS_DYN_CORES +export OPENBLAS_DYN_CORES := $(sort $(DYNAMIC_CORE)) +HASH := \# +DUMMY := $(shell echo '$(HASH)define OPENBLAS_CORE_LIST(X, arg) $(foreach c,$(OPENBLAS_DYN_CORES),X($(c), arg))' > $(TOPDIR)/dyn_cores.h) +endif +endif + ifeq ($(ARCH), ia64) NO_BINARY_MODE = 1 BINARY_DEFINED = 1 diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 12d7de54e7..e74c353204 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -129,9 +129,26 @@ if (DYNAMIC_ARCH) message (FATAL_ERROR "Your build directory contains a file config_kernel.h, probably from a previous compilation with make. This will conflict with the cmake compilation and cause strange compiler errors - please remove the file before trying again") endif () + if (EXISTS ${PROJECT_SOURCE_DIR}/dyn_cores.h) + message (FATAL_ERROR "Your source directory contains a file dyn_cores.h, probably from a previous compilation with make. It would be used instead of the one generated for this build - please remove the file before trying again") + endif () + if (NOT DYNAMIC_CORE) message (STATUS "DYNAMIC_ARCH is not supported on this architecture, removing from options") unset(DYNAMIC_ARCH CACHE) + else () + # dyn_cores.h tells common_param.h which cores this build has; + # see the equivalent rule in Makefile.system. + set(DYN_CORES_LIST ${DYNAMIC_CORE}) + list(REMOVE_DUPLICATES DYN_CORES_LIST) + list(SORT DYN_CORES_LIST) + set(DYN_CORES_H "#define OPENBLAS_CORE_LIST(X, arg)") + foreach (DCORE ${DYN_CORES_LIST}) + string(APPEND DYN_CORES_H " X(${DCORE}, arg)") + endforeach () + file(WRITE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp "${DYN_CORES_H}\n") + configure_file(${PROJECT_BINARY_DIR}/dyn_cores.h.tmp ${PROJECT_BINARY_DIR}/dyn_cores.h COPYONLY) + file(REMOVE ${PROJECT_BINARY_DIR}/dyn_cores.h.tmp) endif () endif () diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 6ae74322dd..0987456210 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1601,7 +1601,7 @@ endif () add_library(kernel${TSUFFIX} OBJECT ${OPENBLAS_SRC}) set_target_properties(kernel${TSUFFIX} PROPERTIES COMPILE_FLAGS "${KERNEL_DEFINITIONS}") get_target_property(KERNEL_INCLUDE_DIRECTORIES kernel${TSUFFIX} INCLUDE_DIRECTORIES) - set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR}") + set_target_properties(kernel${TSUFFIX} PROPERTIES INCLUDE_DIRECTORIES "${KERNEL_INCLUDE_DIRECTORIES};${TARGET_CONF_DIR};${PROJECT_BINARY_DIR}") if (USE_GEMM3M) target_compile_definitions(kernel${TSUFFIX} PRIVATE USE_GEMM3M) endif() From 5e556898ddc7ab7c0cc97fc9357b1235cda24f74 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:18:16 +0000 Subject: [PATCH 3/7] DYNAMIC_ARCH: number the cores, and add per-group dispatch macros Preparation for splitting the kernels out of gotoblas_t; nothing uses the new macros yet. * common_param.h numbers the cores of the build, OPENBLAS_CORE_, from OPENBLAS_CORE_LIST() in dyn_cores.h, and every core's gotoblas_t records its number in the new "core" field. * OPENBLAS_DISPATCH(group) is the running core's table for a group of kernels, from the array openblas__dispatch[], indexed by gotoblas->core. driver/others/dispatch.c will define those arrays, with DEFINE_DISPATCH_TABLE(group). OPENBLAS_DISPATCH_OFFSET/_BASE do for the offset tables of interface/gemm*.c what FUNC_OFFSET/FUNC_BASE do today. Co-Authored-By: Claude Opus 5.5 --- common_param.h | 13 ++++++++++ driver/others/CMakeLists.txt | 1 + driver/others/Makefile | 4 +++ driver/others/dispatch.c | 49 ++++++++++++++++++++++++++++++++++++ kernel/setparam-ref.c | 2 ++ 5 files changed, 69 insertions(+) create mode 100644 driver/others/dispatch.c diff --git a/common_param.h b/common_param.h index 83f9ea3b13..0bc504bdbb 100644 --- a/common_param.h +++ b/common_param.h @@ -44,7 +44,19 @@ #ifdef DYNAMIC_ARCH +/* OPENBLAS_CORE_LIST(X, arg) expands to X(CORE, arg) for each core in this + build. dyn_cores.h is generated from DYNAMIC_CORE by the build system. */ +#include "dyn_cores.h" + +#define OPENBLAS_CORE_ENUM(core, unused) OPENBLAS_CORE_##core, +enum { OPENBLAS_CORE_LIST(OPENBLAS_CORE_ENUM, ~) OPENBLAS_NUM_CORES }; + +#define OPENBLAS_DISPATCH(group) (openblas_##group##_dispatch[gotoblas->core]) +#define OPENBLAS_DISPATCH_OFFSET(group, func) (size_t)(&((openblas_##group##_dispatch_t *)NULL)->func) +#define OPENBLAS_DISPATCH_BASE(group) ((char *)OPENBLAS_DISPATCH(group)) + typedef struct { + int core; int dtb_entries; int switch_ratio; int divide_rate; @@ -1463,6 +1475,7 @@ extern gotoblas_t *gotoblas; #else #define FUNC_OFFSET(func) (size_t)(func) +#define OPENBLAS_DISPATCH_OFFSET(group, func) (size_t)(func) #define DTB_ENTRIES DTB_DEFAULT_ENTRIES diff --git a/driver/others/CMakeLists.txt b/driver/others/CMakeLists.txt index b25337db13..e06b2adafa 100644 --- a/driver/others/CMakeLists.txt +++ b/driver/others/CMakeLists.txt @@ -51,6 +51,7 @@ GenerateNamedObjects("abs.c" "DOUBLE" "z_abs" 0 "" "" 1) GenerateNamedObjects("openblas_get_config.c;openblas_get_parallel.c" "" "" 0 "" "" 1) if (DYNAMIC_ARCH) + list(APPEND COMMON_SOURCES dispatch.c) if (ARM64) list(APPEND COMMON_SOURCES dynamic_arm64.c) elseif (POWER) diff --git a/driver/others/Makefile b/driver/others/Makefile index 52a893e7bf..a8e1428506 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -15,6 +15,7 @@ endif # COMMONOBJS += info.$(SUFFIX) ifeq ($(DYNAMIC_ARCH), 1) +COMMONOBJS += dispatch.$(SUFFIX) ifeq ($(ARCH),arm64) COMMONOBJS += dynamic_arm64.$(SUFFIX) else @@ -133,6 +134,9 @@ dynamic%$(SUFFIX) : dynamic%c dynamic%$(PSUFFIX) : dynamic%c $(CC) $(PFLAGS) -c $< -o $(@F) +dispatch.$(SUFFIX) : dispatch.c + $(CC) $(CFLAGS) -c $< -o $(@F) + parameter.$(SUFFIX) : parameter.c ../../param.h $(CC) $(CFLAGS) -c $< -o $(@F) diff --git a/driver/others/dispatch.c b/driver/others/dispatch.c new file mode 100644 index 0000000000..cecde58205 --- /dev/null +++ b/driver/others/dispatch.c @@ -0,0 +1,49 @@ +/***************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* The arrays of per-core kernel tables that OPENBLAS_DISPATCH(group) + indexes (see common_param.h), one per kernel group. */ + +#include "common.h" + +#define DISPATCH_EXTERN(core, group) \ + extern const openblas_##group##_dispatch_t openblas_##group##_dispatch_##core; +#define DISPATCH_ENTRY(core, group) \ + [OPENBLAS_CORE_##core] = &openblas_##group##_dispatch_##core, + +/* openblas__dispatch[OPENBLAS_CORE_] points to + openblas__dispatch_, which kernel/setparam-ref.c defines for + that core. */ +#define DEFINE_DISPATCH_TABLE(group) \ + OPENBLAS_CORE_LIST(DISPATCH_EXTERN, group) \ + const openblas_##group##_dispatch_t *const openblas_##group##_dispatch[OPENBLAS_NUM_CORES] = { \ + OPENBLAS_CORE_LIST(DISPATCH_ENTRY, group) \ + }; diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 6993062ed2..60fc4c2921 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -51,6 +51,8 @@ extern char* gotoblas_corename(void); static void init_parameter(void); gotoblas_t TABLE_NAME = { + OPENBLAS_CORETS, + DTB_DEFAULT_ENTRIES, SWITCH_RATIO, From f78159e18da07a8eb93d6803a8cdebaa7bccc98b Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 19:42:49 +0000 Subject: [PATCH 4/7] DYNAMIC_ARCH: build with -ffunction-sections -fdata-sections Preparation for splitting the kernels out of gotoblas_t, later in this series. After that split, each core's kernel tables are defined together in setparam_.o, and all the per-group arrays together in dispatch.o; a static link would keep all of them as soon as it kept any, and with them every kernel. Separate sections let --gc-sections keep only the groups a program calls. On its own this changes no static link size: every kernel is still reachable from gotoblas_t. Only for compilers known to accept the flags, and not for MSVC-style frontends. Co-Authored-By: Claude Opus 5.5 --- Makefile.system | 3 +++ cmake/system.cmake | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Makefile.system b/Makefile.system index 9c89611d7a..6c0c4ae0de 100644 --- a/Makefile.system +++ b/Makefile.system @@ -1493,6 +1493,9 @@ endif ifeq ($(DYNAMIC_ARCH), 1) CCOMMON_OPT += -DDYNAMIC_ARCH +ifeq ($(C_COMPILER), $(filter $(C_COMPILER),GCC LSB CLANG INTEL)) +CCOMMON_OPT += -ffunction-sections -fdata-sections +endif endif ifeq ($(DYNAMIC_OLDER), 1) diff --git a/cmake/system.cmake b/cmake/system.cmake index 436f2109aa..5953d27301 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -536,6 +536,9 @@ endif () if (DYNAMIC_ARCH) if (X86 OR X86_64 OR ARM64 OR POWER OR RISCV64 OR LOONGARCH64 OR ZARCH) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH") + if (CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|Intel" AND NOT MSVC) + set(CCOMMON_OPT "${CCOMMON_OPT} -ffunction-sections -fdata-sections") + endif () if (DYNAMIC_OLDER) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER") endif () From 297da0b40228b1d4bb4c3d3739e0e8b99e46ab29 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 20:45:25 +0000 Subject: [PATCH 5/7] setparam-ref.c: use designated initializers for gotoblas_t No functional change. The gotoblas_t initializer was positional across roughly 1300 entries and two dozen nested #if blocks, so which value lands in which field could only be worked out by lining it up against the struct definition by hand. Label every entry with its field, one entry per line. Generated by dispatch-split/add_explicit_inits.py, which pairs the entries with the fields of gotoblas_t for every combination of the macros that guard those fields (BUILD_*, SMALL_MATRIX_OPT, EXPRECISION, ARCH_*: 1024 configurations) and stops if any entry would get a different field in any of them. setparam_.o is byte-identical before and after for all 14 x86_64 DYNAMIC_ARCH cores, both as built and with BFLOAT16/HFLOAT16 enabled, EXPRECISION and SMALL_MATRIX_OPT disabled, or only one of BUILD_SINGLE, BUILD_DOUBLE, BUILD_COMPLEX and BUILD_COMPLEX16 enabled. Generated-by: dispatch-split/add_explicit_inits.py Co-Authored-By: Claude Opus 5.5 --- kernel/setparam-ref.c | 2038 ++++++++++++++++++++++++++++------------- 1 file changed, 1426 insertions(+), 612 deletions(-) diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 60fc4c2921..65a17733dc 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -51,941 +51,1755 @@ extern char* gotoblas_corename(void); static void init_parameter(void); gotoblas_t TABLE_NAME = { - OPENBLAS_CORETS, + .core = OPENBLAS_CORETS, - DTB_DEFAULT_ENTRIES, + .dtb_entries = DTB_DEFAULT_ENTRIES, - SWITCH_RATIO, + .switch_ratio = SWITCH_RATIO, - GEMM_DIVIDE_RATE, + .divide_rate = GEMM_DIVIDE_RATE, - GEMM_DIVIDE_LIMIT, + .divide_limit = GEMM_DIVIDE_LIMIT, - GEMM_PREFERRED_SIZE, + .preferred_size = GEMM_PREFERRED_SIZE, - GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN, + .offsetA = GEMM_DEFAULT_OFFSET_A, + .offsetB = GEMM_DEFAULT_OFFSET_B, + .align = GEMM_DEFAULT_ALIGN, #ifdef BUILD_HFLOAT16 - 0, 0, 0, - SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N, + .shgemm_p = 0, + .shgemm_q = 0, + .shgemm_r = 0, + .shgemm_unroll_m = SHGEMM_DEFAULT_UNROLL_M, + .shgemm_unroll_n = SHGEMM_DEFAULT_UNROLL_N, #ifdef SHGEMM_DEFAULT_UNROLL_MN - SHGEMM_DEFAULT_UNROLL_MN, + .shgemm_unroll_mn = SHGEMM_DEFAULT_UNROLL_MN, #else - MAX(SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N), + .shgemm_unroll_mn = MAX(SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N), #endif - shgemm_kernelTS, shgemm_betaTS, + .shgemm_kernel = shgemm_kernelTS, + .shgemm_beta = shgemm_betaTS, #if SHGEMM_DEFAULT_UNROLL_M != SHGEMM_DEFAULT_UNROLL_N - shgemm_incopyTS, shgemm_itcopyTS, + .shgemm_incopy = shgemm_incopyTS, + .shgemm_itcopy = shgemm_itcopyTS, #else - shgemm_oncopyTS, shgemm_otcopyTS, + .shgemm_incopy = shgemm_oncopyTS, + .shgemm_itcopy = shgemm_otcopyTS, #endif - shgemm_oncopyTS, shgemm_otcopyTS, - shgemv_nTS, shgemv_tTS, + .shgemm_oncopy = shgemm_oncopyTS, + .shgemm_otcopy = shgemm_otcopyTS, + .shgemv_n = shgemv_nTS, + .shgemv_t = shgemv_tTS, #endif #ifdef BUILD_BFLOAT16 - 0, 0, 0, - BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N, + .bgemm_p = 0, + .bgemm_q = 0, + .bgemm_r = 0, + .bgemm_unroll_m = BGEMM_DEFAULT_UNROLL_M, + .bgemm_unroll_n = BGEMM_DEFAULT_UNROLL_N, #ifdef BGEMM_DEFAULT_UNROLL_MN - BGEMM_DEFAULT_UNROLL_MN, + .bgemm_unroll_mn = BGEMM_DEFAULT_UNROLL_MN, #else - MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N), + .bgemm_unroll_mn = MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N), #endif - BGEMM_ALIGN_K, + .bgemm_align_k = BGEMM_ALIGN_K, - 0, 0, 0, - SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N, + .sbgemm_p = 0, + .sbgemm_q = 0, + .sbgemm_r = 0, + .sbgemm_unroll_m = SBGEMM_DEFAULT_UNROLL_M, + .sbgemm_unroll_n = SBGEMM_DEFAULT_UNROLL_N, #ifdef SBGEMM_DEFAULT_UNROLL_MN - SBGEMM_DEFAULT_UNROLL_MN, -#else - MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N), -#endif - - SBGEMM_ALIGN_K, - 0, // need_amxtile_permission - - sbstobf16_kTS, sbdtobf16_kTS, sbf16tos_kTS, dbf16tod_kTS, - - samax_kTS, samin_kTS, smax_kTS, smin_kTS, - isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS, - snrm2_kTS, sasum_kTS, ssum_kTS, scopy_kTS, sbdot_kTS, - dsdot_kTS, - srot_kTS, srotm_kTS, bscal_kTS, saxpy_kTS, sscal_kTS, sswap_kTS, - bgemv_nTS, bgemv_tTS, sbgemv_nTS, sbgemv_tTS, sger_kTS, - ssymv_LTS, ssymv_UTS, - - bgemm_kernelTS, bgemm_betaTS, + .sbgemm_unroll_mn = SBGEMM_DEFAULT_UNROLL_MN, +#else + .sbgemm_unroll_mn = MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N), +#endif + + .sbgemm_align_k = SBGEMM_ALIGN_K, + .need_amxtile_permission = 0, // need_amxtile_permission + + .sbstobf16_k = sbstobf16_kTS, + .sbdtobf16_k = sbdtobf16_kTS, + .sbf16tos_k = sbf16tos_kTS, + .dbf16tod_k = dbf16tod_kTS, + + .sbamax_k = samax_kTS, + .sbamin_k = samin_kTS, + .sbmax_k = smax_kTS, + .sbmin_k = smin_kTS, + .isbamax_k = isamax_kTS, + .isbamin_k = isamin_kTS, + .isbmax_k = ismax_kTS, + .isbmin_k = ismin_kTS, + .sbnrm2_k = snrm2_kTS, + .sbasum_k = sasum_kTS, + .sbsum_k = ssum_kTS, + .sbcopy_k = scopy_kTS, + .sbdot_k = sbdot_kTS, + .dsbdot_k = dsdot_kTS, + .sbrot_k = srot_kTS, + .sbrotm_k = srotm_kTS, + .bscal_k = bscal_kTS, + .sbaxpy_k = saxpy_kTS, + .sbscal_k = sscal_kTS, + .sbswap_k = sswap_kTS, + .bgemv_n = bgemv_nTS, + .bgemv_t = bgemv_tTS, + .sbgemv_n = sbgemv_nTS, + .sbgemv_t = sbgemv_tTS, + .sbger_k = sger_kTS, + .sbsymv_L = ssymv_LTS, + .sbsymv_U = ssymv_UTS, + + .bgemm_kernel = bgemm_kernelTS, + .bgemm_beta = bgemm_betaTS, #if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N - bgemm_incopyTS, bgemm_itcopyTS, + .bgemm_incopy = bgemm_incopyTS, + .bgemm_itcopy = bgemm_itcopyTS, #else - bgemm_oncopyTS, bgemm_otcopyTS, + .bgemm_incopy = bgemm_oncopyTS, + .bgemm_itcopy = bgemm_otcopyTS, #endif - bgemm_oncopyTS, bgemm_otcopyTS, + .bgemm_oncopy = bgemm_oncopyTS, + .bgemm_otcopy = bgemm_otcopyTS, - sbgemm_kernelTS, sbgemm_betaTS, + .sbgemm_kernel = sbgemm_kernelTS, + .sbgemm_beta = sbgemm_betaTS, #if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N - sbgemm_incopyTS, sbgemm_itcopyTS, + .sbgemm_incopy = sbgemm_incopyTS, + .sbgemm_itcopy = sbgemm_itcopyTS, #else - sbgemm_oncopyTS, sbgemm_otcopyTS, + .sbgemm_incopy = sbgemm_oncopyTS, + .sbgemm_itcopy = sbgemm_otcopyTS, #endif - sbgemm_oncopyTS, sbgemm_otcopyTS, + .sbgemm_oncopy = sbgemm_oncopyTS, + .sbgemm_otcopy = sbgemm_otcopyTS, - strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS, + .sbtrsm_kernel_LN = strsm_kernel_LNTS, + .sbtrsm_kernel_LT = strsm_kernel_LTTS, + .sbtrsm_kernel_RN = strsm_kernel_RNTS, + .sbtrsm_kernel_RT = strsm_kernel_RTTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS, - strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS, -#else - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, -#endif - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, - strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS, + .sbtrsm_iunucopy = strsm_iunucopyTS, + .sbtrsm_iunncopy = strsm_iunncopyTS, + .sbtrsm_iutucopy = strsm_iutucopyTS, + .sbtrsm_iutncopy = strsm_iutncopyTS, + .sbtrsm_ilnucopy = strsm_ilnucopyTS, + .sbtrsm_ilnncopy = strsm_ilnncopyTS, + .sbtrsm_iltucopy = strsm_iltucopyTS, + .sbtrsm_iltncopy = strsm_iltncopyTS, +#else + .sbtrsm_iunucopy = strsm_ounucopyTS, + .sbtrsm_iunncopy = strsm_ounncopyTS, + .sbtrsm_iutucopy = strsm_outucopyTS, + .sbtrsm_iutncopy = strsm_outncopyTS, + .sbtrsm_ilnucopy = strsm_olnucopyTS, + .sbtrsm_ilnncopy = strsm_olnncopyTS, + .sbtrsm_iltucopy = strsm_oltucopyTS, + .sbtrsm_iltncopy = strsm_oltncopyTS, +#endif + .sbtrsm_ounucopy = strsm_ounucopyTS, + .sbtrsm_ounncopy = strsm_ounncopyTS, + .sbtrsm_outucopy = strsm_outucopyTS, + .sbtrsm_outncopy = strsm_outncopyTS, + .sbtrsm_olnucopy = strsm_olnucopyTS, + .sbtrsm_olnncopy = strsm_olnncopyTS, + .sbtrsm_oltucopy = strsm_oltucopyTS, + .sbtrsm_oltncopy = strsm_oltncopyTS, + .sbtrmm_kernel_RN = strmm_kernel_RNTS, + .sbtrmm_kernel_RT = strmm_kernel_RTTS, + .sbtrmm_kernel_LN = strmm_kernel_LNTS, + .sbtrmm_kernel_LT = strmm_kernel_LTTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS, - strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS, -#else - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, -#endif - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, + .sbtrmm_iunucopy = strmm_iunucopyTS, + .sbtrmm_iunncopy = strmm_iunncopyTS, + .sbtrmm_iutucopy = strmm_iutucopyTS, + .sbtrmm_iutncopy = strmm_iutncopyTS, + .sbtrmm_ilnucopy = strmm_ilnucopyTS, + .sbtrmm_ilnncopy = strmm_ilnncopyTS, + .sbtrmm_iltucopy = strmm_iltucopyTS, + .sbtrmm_iltncopy = strmm_iltncopyTS, +#else + .sbtrmm_iunucopy = strmm_ounucopyTS, + .sbtrmm_iunncopy = strmm_ounncopyTS, + .sbtrmm_iutucopy = strmm_outucopyTS, + .sbtrmm_iutncopy = strmm_outncopyTS, + .sbtrmm_ilnucopy = strmm_olnucopyTS, + .sbtrmm_ilnncopy = strmm_olnncopyTS, + .sbtrmm_iltucopy = strmm_oltucopyTS, + .sbtrmm_iltncopy = strmm_oltncopyTS, +#endif + .sbtrmm_ounucopy = strmm_ounucopyTS, + .sbtrmm_ounncopy = strmm_ounncopyTS, + .sbtrmm_outucopy = strmm_outucopyTS, + .sbtrmm_outncopy = strmm_outncopyTS, + .sbtrmm_olnucopy = strmm_olnucopyTS, + .sbtrmm_olnncopy = strmm_olnncopyTS, + .sbtrmm_oltucopy = strmm_oltucopyTS, + .sbtrmm_oltncopy = strmm_oltncopyTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - ssymm_iutcopyTS, ssymm_iltcopyTS, + .sbsymm_iutcopy = ssymm_iutcopyTS, + .sbsymm_iltcopy = ssymm_iltcopyTS, #else - ssymm_outcopyTS, ssymm_oltcopyTS, + .sbsymm_iutcopy = ssymm_outcopyTS, + .sbsymm_iltcopy = ssymm_oltcopyTS, #endif - ssymm_outcopyTS, ssymm_oltcopyTS, + .sbsymm_outcopy = ssymm_outcopyTS, + .sbsymm_oltcopy = ssymm_oltcopyTS, #ifndef NO_LAPACK - sneg_tcopyTS, slaswp_ncopyTS, + .sbneg_tcopy = sneg_tcopyTS, + .sblaswp_ncopy = slaswp_ncopyTS, #else - NULL,NULL, + .sbneg_tcopy = NULL, + .sblaswp_ncopy = NULL, #endif #ifdef SMALL_MATRIX_OPT - sbgemm_small_matrix_permitTS, - sbgemm_small_kernel_nnTS, sbgemm_small_kernel_ntTS, sbgemm_small_kernel_tnTS, sbgemm_small_kernel_ttTS, - sbgemm_small_kernel_b0_nnTS, sbgemm_small_kernel_b0_ntTS, sbgemm_small_kernel_b0_tnTS, sbgemm_small_kernel_b0_ttTS, + .sbgemm_small_matrix_permit = sbgemm_small_matrix_permitTS, + .sbgemm_small_kernel_nn = sbgemm_small_kernel_nnTS, + .sbgemm_small_kernel_nt = sbgemm_small_kernel_ntTS, + .sbgemm_small_kernel_tn = sbgemm_small_kernel_tnTS, + .sbgemm_small_kernel_tt = sbgemm_small_kernel_ttTS, + .sbgemm_small_kernel_b0_nn = sbgemm_small_kernel_b0_nnTS, + .sbgemm_small_kernel_b0_nt = sbgemm_small_kernel_b0_ntTS, + .sbgemm_small_kernel_b0_tn = sbgemm_small_kernel_b0_tnTS, + .sbgemm_small_kernel_b0_tt = sbgemm_small_kernel_b0_ttTS, #endif #endif #if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) - 0, 0, 0, - SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, + .sgemm_p = 0, + .sgemm_q = 0, + .sgemm_r = 0, + .sgemm_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .sgemm_unroll_n = SGEMM_DEFAULT_UNROLL_N, #ifdef SGEMM_DEFAULT_UNROLL_MN - SGEMM_DEFAULT_UNROLL_MN, + .sgemm_unroll_mn = SGEMM_DEFAULT_UNROLL_MN, #else - MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), + .sgemm_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), #endif #endif #ifdef HAVE_EXCLUSIVE_CACHE - 1, + .exclusive_cache = 1, #else - 0, + .exclusive_cache = 0, #endif #if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) - samax_kTS, samin_kTS, smax_kTS, smin_kTS, + .samax_k = samax_kTS, + .samin_k = samin_kTS, + .smax_k = smax_kTS, + .smin_k = smin_kTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - isamax_kTS, + .isamax_k = isamax_kTS, #endif #if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) - isamin_kTS, ismax_kTS, ismin_kTS, - snrm2_kTS, sasum_kTS, + .isamin_k = isamin_kTS, + .ismax_k = ismax_kTS, + .ismin_k = ismin_kTS, + .snrm2_k = snrm2_kTS, + .sasum_k = sasum_kTS, #endif #if BUILD_SINGLE == 1 - ssum_kTS, + .ssum_k = ssum_kTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - scopy_kTS, sdot_kTS, + .scopy_k = scopy_kTS, + .sdot_k = sdot_kTS, // dsdot_kTS, - srot_kTS, srotm_kTS, saxpy_kTS, + .srot_k = srot_kTS, + .srotm_k = srotm_kTS, + .saxpy_k = saxpy_kTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) - sscal_kTS, + .sscal_k = sscal_kTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) - sswap_kTS, - sgemv_nTS, sgemv_tTS, + .sswap_k = sswap_kTS, + .sgemv_n = sgemv_nTS, + .sgemv_t = sgemv_tTS, #endif #if BUILD_SINGLE == 1 - sger_kTS, + .sger_k = sger_kTS, #endif #if BUILD_SINGLE == 1 - ssymv_LTS, ssymv_UTS, + .ssymv_L = ssymv_LTS, + .ssymv_U = ssymv_UTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) #ifdef ARCH_X86_64 - sgemm_directTS, - sgemm_direct_performantTS, + .sgemm_direct = sgemm_directTS, + .sgemm_direct_performant = sgemm_direct_performantTS, #endif #ifdef ARCH_ARM64 - sgemm_directTS, - sgemm_direct_performantTS, - sgemm_direct_alpha_betaTS, - ssymm_direct_alpha_betaLUTS, - ssymm_direct_alpha_betaLLTS, - strmm_direct_LNUNTS, - strmm_direct_LNLNTS, - strmm_direct_LTUNTS, - strmm_direct_LTLNTS, - ssyrk_direct_alpha_betaUNTS, - ssyrk_direct_alpha_betaUTTS, - ssyrk_direct_alpha_betaLNTS, - ssyrk_direct_alpha_betaLTTS, - ssyr2k_direct_alpha_betaUNTS, - ssyr2k_direct_alpha_betaUTTS, - ssyr2k_direct_alpha_betaLNTS, - ssyr2k_direct_alpha_betaLTTS, + .sgemm_direct = sgemm_directTS, + .sgemm_direct_performant = sgemm_direct_performantTS, + .sgemm_direct_alpha_beta = sgemm_direct_alpha_betaTS, + .ssymm_direct_alpha_betaLU = ssymm_direct_alpha_betaLUTS, + .ssymm_direct_alpha_betaLL = ssymm_direct_alpha_betaLLTS, + .strmm_direct_LNUN = strmm_direct_LNUNTS, + .strmm_direct_LNLN = strmm_direct_LNLNTS, + .strmm_direct_LTUN = strmm_direct_LTUNTS, + .strmm_direct_LTLN = strmm_direct_LTLNTS, + .ssyrk_direct_alpha_betaUN = ssyrk_direct_alpha_betaUNTS, + .ssyrk_direct_alpha_betaUT = ssyrk_direct_alpha_betaUTTS, + .ssyrk_direct_alpha_betaLN = ssyrk_direct_alpha_betaLNTS, + .ssyrk_direct_alpha_betaLT = ssyrk_direct_alpha_betaLTTS, + .ssyr2k_direct_alpha_betaUN = ssyr2k_direct_alpha_betaUNTS, + .ssyr2k_direct_alpha_betaUT = ssyr2k_direct_alpha_betaUTTS, + .ssyr2k_direct_alpha_betaLN = ssyr2k_direct_alpha_betaLNTS, + .ssyr2k_direct_alpha_betaLT = ssyr2k_direct_alpha_betaLTTS, #ifdef HAVE_SME - sme_sgemm_kernelTS, + .sme_sgemm_kernel = sme_sgemm_kernelTS, #else - NULL, + .sme_sgemm_kernel = NULL, #endif #endif - sgemm_kernelTS, sgemm_betaTS, ssymm_kernelTS, strmm_gemm_kernelTS, + .sgemm_kernel = sgemm_kernelTS, + .sgemm_beta = sgemm_betaTS, + .ssymm_kernel = ssymm_kernelTS, + .strmm_gemm_kernel = strmm_gemm_kernelTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - sgemm_incopyTS, sgemm_itcopyTS, + .sgemm_incopy = sgemm_incopyTS, + .sgemm_itcopy = sgemm_itcopyTS, #else - sgemm_oncopyTS, sgemm_otcopyTS, + .sgemm_incopy = sgemm_oncopyTS, + .sgemm_itcopy = sgemm_otcopyTS, #endif - sgemm_oncopyTS, sgemm_otcopyTS, + .sgemm_oncopy = sgemm_oncopyTS, + .sgemm_otcopy = sgemm_otcopyTS, #endif #if BUILD_SINGLE == 1 || BUILD_DOUBLE == 1 || BUILD_COMPLEX == 1 #ifdef SMALL_MATRIX_OPT - sgemm_small_matrix_permitTS, - sgemm_small_kernel_nnTS, sgemm_small_kernel_ntTS, sgemm_small_kernel_tnTS, sgemm_small_kernel_ttTS, - sgemm_small_kernel_b0_nnTS, sgemm_small_kernel_b0_ntTS, sgemm_small_kernel_b0_tnTS, sgemm_small_kernel_b0_ttTS, + .sgemm_small_matrix_permit = sgemm_small_matrix_permitTS, + .sgemm_small_kernel_nn = sgemm_small_kernel_nnTS, + .sgemm_small_kernel_nt = sgemm_small_kernel_ntTS, + .sgemm_small_kernel_tn = sgemm_small_kernel_tnTS, + .sgemm_small_kernel_tt = sgemm_small_kernel_ttTS, + .sgemm_small_kernel_b0_nn = sgemm_small_kernel_b0_nnTS, + .sgemm_small_kernel_b0_nt = sgemm_small_kernel_b0_ntTS, + .sgemm_small_kernel_b0_tn = sgemm_small_kernel_b0_tnTS, + .sgemm_small_kernel_b0_tt = sgemm_small_kernel_b0_ttTS, #endif #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX == 1) - strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS, + .strsm_kernel_LN = strsm_kernel_LNTS, + .strsm_kernel_LT = strsm_kernel_LTTS, + .strsm_kernel_RN = strsm_kernel_RNTS, + .strsm_kernel_RT = strsm_kernel_RTTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strsm_iunucopyTS, strsm_iunncopyTS, strsm_iutucopyTS, strsm_iutncopyTS, - strsm_ilnucopyTS, strsm_ilnncopyTS, strsm_iltucopyTS, strsm_iltncopyTS, -#else - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, -#endif - strsm_ounucopyTS, strsm_ounncopyTS, strsm_outucopyTS, strsm_outncopyTS, - strsm_olnucopyTS, strsm_olnncopyTS, strsm_oltucopyTS, strsm_oltncopyTS, + .strsm_iunucopy = strsm_iunucopyTS, + .strsm_iunncopy = strsm_iunncopyTS, + .strsm_iutucopy = strsm_iutucopyTS, + .strsm_iutncopy = strsm_iutncopyTS, + .strsm_ilnucopy = strsm_ilnucopyTS, + .strsm_ilnncopy = strsm_ilnncopyTS, + .strsm_iltucopy = strsm_iltucopyTS, + .strsm_iltncopy = strsm_iltncopyTS, +#else + .strsm_iunucopy = strsm_ounucopyTS, + .strsm_iunncopy = strsm_ounncopyTS, + .strsm_iutucopy = strsm_outucopyTS, + .strsm_iutncopy = strsm_outncopyTS, + .strsm_ilnucopy = strsm_olnucopyTS, + .strsm_ilnncopy = strsm_olnncopyTS, + .strsm_iltucopy = strsm_oltucopyTS, + .strsm_iltncopy = strsm_oltncopyTS, +#endif + .strsm_ounucopy = strsm_ounucopyTS, + .strsm_ounncopy = strsm_ounncopyTS, + .strsm_outucopy = strsm_outucopyTS, + .strsm_outncopy = strsm_outncopyTS, + .strsm_olnucopy = strsm_olnucopyTS, + .strsm_olnncopy = strsm_olnncopyTS, + .strsm_oltucopy = strsm_oltucopyTS, + .strsm_oltncopy = strsm_oltncopyTS, #endif #if (BUILD_SINGLE==1) - strmm_kernel_RNTS, strmm_kernel_RTTS, strmm_kernel_LNTS, strmm_kernel_LTTS, + .strmm_kernel_RN = strmm_kernel_RNTS, + .strmm_kernel_RT = strmm_kernel_RTTS, + .strmm_kernel_LN = strmm_kernel_LNTS, + .strmm_kernel_LT = strmm_kernel_LTTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - strmm_iunucopyTS, strmm_iunncopyTS, strmm_iutucopyTS, strmm_iutncopyTS, - strmm_ilnucopyTS, strmm_ilnncopyTS, strmm_iltucopyTS, strmm_iltncopyTS, -#else - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, -#endif - ssymm_incopyTS, ssymm_itcopyTS, - strmm_incopyTS, strmm_itcopyTS, - strmm_ounucopyTS, strmm_ounncopyTS, strmm_outucopyTS, strmm_outncopyTS, - strmm_olnucopyTS, strmm_olnncopyTS, strmm_oltucopyTS, strmm_oltncopyTS, + .strmm_iunucopy = strmm_iunucopyTS, + .strmm_iunncopy = strmm_iunncopyTS, + .strmm_iutucopy = strmm_iutucopyTS, + .strmm_iutncopy = strmm_iutncopyTS, + .strmm_ilnucopy = strmm_ilnucopyTS, + .strmm_ilnncopy = strmm_ilnncopyTS, + .strmm_iltucopy = strmm_iltucopyTS, + .strmm_iltncopy = strmm_iltncopyTS, +#else + .strmm_iunucopy = strmm_ounucopyTS, + .strmm_iunncopy = strmm_ounncopyTS, + .strmm_iutucopy = strmm_outucopyTS, + .strmm_iutncopy = strmm_outncopyTS, + .strmm_ilnucopy = strmm_olnucopyTS, + .strmm_ilnncopy = strmm_olnncopyTS, + .strmm_iltucopy = strmm_oltucopyTS, + .strmm_iltncopy = strmm_oltncopyTS, +#endif + .ssymm_incopy = ssymm_incopyTS, + .ssymm_itcopy = ssymm_itcopyTS, + .strmm_incopy = strmm_incopyTS, + .strmm_itcopy = strmm_itcopyTS, + .strmm_ounucopy = strmm_ounucopyTS, + .strmm_ounncopy = strmm_ounncopyTS, + .strmm_outucopy = strmm_outucopyTS, + .strmm_outncopy = strmm_outncopyTS, + .strmm_olnucopy = strmm_olnucopyTS, + .strmm_olnncopy = strmm_olnncopyTS, + .strmm_oltucopy = strmm_oltucopyTS, + .strmm_oltncopy = strmm_oltncopyTS, #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - ssymm_iutcopyTS, ssymm_iltcopyTS, + .ssymm_iutcopy = ssymm_iutcopyTS, + .ssymm_iltcopy = ssymm_iltcopyTS, #else - ssymm_outcopyTS, ssymm_oltcopyTS, + .ssymm_iutcopy = ssymm_outcopyTS, + .ssymm_iltcopy = ssymm_oltcopyTS, #endif - ssymm_outcopyTS, ssymm_oltcopyTS, + .ssymm_outcopy = ssymm_outcopyTS, + .ssymm_oltcopy = ssymm_oltcopyTS, #ifndef NO_LAPACK - sneg_tcopyTS, slaswp_ncopyTS, + .sneg_tcopy = sneg_tcopyTS, + .slaswp_ncopy = slaswp_ncopyTS, #else - NULL,NULL, + .sneg_tcopy = NULL, + .slaswp_ncopy = NULL, #endif #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - 0, 0, 0, - DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, + .dgemm_p = 0, + .dgemm_q = 0, + .dgemm_r = 0, + .dgemm_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .dgemm_unroll_n = DGEMM_DEFAULT_UNROLL_N, #ifdef DGEMM_DEFAULT_UNROLL_MN - DGEMM_DEFAULT_UNROLL_MN, + .dgemm_unroll_mn = DGEMM_DEFAULT_UNROLL_MN, #else - MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), + .dgemm_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), #endif #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - damax_kTS, damin_kTS, dmax_kTS, dmin_kTS, - idamax_kTS, idamin_kTS, idmax_kTS, idmin_kTS, - dnrm2_kTS, dasum_kTS, + .damax_k = damax_kTS, + .damin_k = damin_kTS, + .dmax_k = dmax_kTS, + .dmin_k = dmin_kTS, + .idamax_k = idamax_kTS, + .idamin_k = idamin_kTS, + .idmax_k = idmax_kTS, + .idmin_k = idmin_kTS, + .dnrm2_k = dnrm2_kTS, + .dasum_k = dasum_kTS, #endif #if (BUILD_DOUBLE==1) - dsum_kTS, + .dsum_k = dsum_kTS, #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - dcopy_kTS, ddot_kTS, + .dcopy_k = dcopy_kTS, + .ddot_k = ddot_kTS, #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) - dsdot_kTS, + .dsdot_k = dsdot_kTS, #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - drot_kTS, - drotm_kTS, - daxpy_kTS, - dscal_kTS, - dswap_kTS, - dgemv_nTS, dgemv_tTS, + .drot_k = drot_kTS, + .drotm_k = drotm_kTS, + .daxpy_k = daxpy_kTS, + .dscal_k = dscal_kTS, + .dswap_k = dswap_kTS, + .dgemv_n = dgemv_nTS, + .dgemv_t = dgemv_tTS, #endif #if (BUILD_DOUBLE==1) - dger_kTS, - dsymv_LTS, dsymv_UTS, + .dger_k = dger_kTS, + .dsymv_L = dsymv_LTS, + .dsymv_U = dsymv_UTS, #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) #ifdef ARCH_ARM64 #ifdef HAVE_SME - sme_dgemm_kernelTS, + .sme_dgemm_kernel = sme_dgemm_kernelTS, #else - NULL, + .sme_dgemm_kernel = NULL, #endif #endif - dgemm_kernelTS, dgemm_betaTS, dsymm_kernelTS, dtrmm_gemm_kernelTS, + .dgemm_kernel = dgemm_kernelTS, + .dgemm_beta = dgemm_betaTS, + .dsymm_kernel = dsymm_kernelTS, + .dtrmm_gemm_kernel = dtrmm_gemm_kernelTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dgemm_incopyTS, dgemm_itcopyTS, + .dgemm_incopy = dgemm_incopyTS, + .dgemm_itcopy = dgemm_itcopyTS, #else - dgemm_oncopyTS, dgemm_otcopyTS, + .dgemm_incopy = dgemm_oncopyTS, + .dgemm_itcopy = dgemm_otcopyTS, #endif - dgemm_oncopyTS, dgemm_otcopyTS, + .dgemm_oncopy = dgemm_oncopyTS, + .dgemm_otcopy = dgemm_otcopyTS, #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) #ifdef SMALL_MATRIX_OPT - dgemm_small_matrix_permitTS, - dgemm_small_kernel_nnTS, dgemm_small_kernel_ntTS, dgemm_small_kernel_tnTS, dgemm_small_kernel_ttTS, - dgemm_small_kernel_b0_nnTS, dgemm_small_kernel_b0_ntTS, dgemm_small_kernel_b0_tnTS, dgemm_small_kernel_b0_ttTS, + .dgemm_small_matrix_permit = dgemm_small_matrix_permitTS, + .dgemm_small_kernel_nn = dgemm_small_kernel_nnTS, + .dgemm_small_kernel_nt = dgemm_small_kernel_ntTS, + .dgemm_small_kernel_tn = dgemm_small_kernel_tnTS, + .dgemm_small_kernel_tt = dgemm_small_kernel_ttTS, + .dgemm_small_kernel_b0_nn = dgemm_small_kernel_b0_nnTS, + .dgemm_small_kernel_b0_nt = dgemm_small_kernel_b0_ntTS, + .dgemm_small_kernel_b0_tn = dgemm_small_kernel_b0_tnTS, + .dgemm_small_kernel_b0_tt = dgemm_small_kernel_b0_ttTS, #endif #endif #if (BUILD_DOUBLE==1) - dtrsm_kernel_LNTS, dtrsm_kernel_LTTS, dtrsm_kernel_RNTS, dtrsm_kernel_RTTS, + .dtrsm_kernel_LN = dtrsm_kernel_LNTS, + .dtrsm_kernel_LT = dtrsm_kernel_LTTS, + .dtrsm_kernel_RN = dtrsm_kernel_RNTS, + .dtrsm_kernel_RT = dtrsm_kernel_RTTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dtrsm_iunucopyTS, dtrsm_iunncopyTS, dtrsm_iutucopyTS, dtrsm_iutncopyTS, - dtrsm_ilnucopyTS, dtrsm_ilnncopyTS, dtrsm_iltucopyTS, dtrsm_iltncopyTS, -#else - dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS, - dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS, -#endif - dtrsm_ounucopyTS, dtrsm_ounncopyTS, dtrsm_outucopyTS, dtrsm_outncopyTS, - dtrsm_olnucopyTS, dtrsm_olnncopyTS, dtrsm_oltucopyTS, dtrsm_oltncopyTS, - dtrmm_kernel_RNTS, dtrmm_kernel_RTTS, dtrmm_kernel_LNTS, dtrmm_kernel_LTTS, + .dtrsm_iunucopy = dtrsm_iunucopyTS, + .dtrsm_iunncopy = dtrsm_iunncopyTS, + .dtrsm_iutucopy = dtrsm_iutucopyTS, + .dtrsm_iutncopy = dtrsm_iutncopyTS, + .dtrsm_ilnucopy = dtrsm_ilnucopyTS, + .dtrsm_ilnncopy = dtrsm_ilnncopyTS, + .dtrsm_iltucopy = dtrsm_iltucopyTS, + .dtrsm_iltncopy = dtrsm_iltncopyTS, +#else + .dtrsm_iunucopy = dtrsm_ounucopyTS, + .dtrsm_iunncopy = dtrsm_ounncopyTS, + .dtrsm_iutucopy = dtrsm_outucopyTS, + .dtrsm_iutncopy = dtrsm_outncopyTS, + .dtrsm_ilnucopy = dtrsm_olnucopyTS, + .dtrsm_ilnncopy = dtrsm_olnncopyTS, + .dtrsm_iltucopy = dtrsm_oltucopyTS, + .dtrsm_iltncopy = dtrsm_oltncopyTS, +#endif + .dtrsm_ounucopy = dtrsm_ounucopyTS, + .dtrsm_ounncopy = dtrsm_ounncopyTS, + .dtrsm_outucopy = dtrsm_outucopyTS, + .dtrsm_outncopy = dtrsm_outncopyTS, + .dtrsm_olnucopy = dtrsm_olnucopyTS, + .dtrsm_olnncopy = dtrsm_olnncopyTS, + .dtrsm_oltucopy = dtrsm_oltucopyTS, + .dtrsm_oltncopy = dtrsm_oltncopyTS, + .dtrmm_kernel_RN = dtrmm_kernel_RNTS, + .dtrmm_kernel_RT = dtrmm_kernel_RTTS, + .dtrmm_kernel_LN = dtrmm_kernel_LNTS, + .dtrmm_kernel_LT = dtrmm_kernel_LTTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dtrmm_iunucopyTS, dtrmm_iunncopyTS, dtrmm_iutucopyTS, dtrmm_iutncopyTS, - dtrmm_ilnucopyTS, dtrmm_ilnncopyTS, dtrmm_iltucopyTS, dtrmm_iltncopyTS, -#else - dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, - dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, -#endif - dsymm_incopyTS, dsymm_itcopyTS, - dtrmm_incopyTS, dtrmm_itcopyTS, - dtrmm_ounucopyTS, dtrmm_ounncopyTS, dtrmm_outucopyTS, dtrmm_outncopyTS, - dtrmm_olnucopyTS, dtrmm_olnncopyTS, dtrmm_oltucopyTS, dtrmm_oltncopyTS, + .dtrmm_iunucopy = dtrmm_iunucopyTS, + .dtrmm_iunncopy = dtrmm_iunncopyTS, + .dtrmm_iutucopy = dtrmm_iutucopyTS, + .dtrmm_iutncopy = dtrmm_iutncopyTS, + .dtrmm_ilnucopy = dtrmm_ilnucopyTS, + .dtrmm_ilnncopy = dtrmm_ilnncopyTS, + .dtrmm_iltucopy = dtrmm_iltucopyTS, + .dtrmm_iltncopy = dtrmm_iltncopyTS, +#else + .dtrmm_iunucopy = dtrmm_ounucopyTS, + .dtrmm_iunncopy = dtrmm_ounncopyTS, + .dtrmm_iutucopy = dtrmm_outucopyTS, + .dtrmm_iutncopy = dtrmm_outncopyTS, + .dtrmm_ilnucopy = dtrmm_olnucopyTS, + .dtrmm_ilnncopy = dtrmm_olnncopyTS, + .dtrmm_iltucopy = dtrmm_oltucopyTS, + .dtrmm_iltncopy = dtrmm_oltncopyTS, +#endif + .dsymm_incopy = dsymm_incopyTS, + .dsymm_itcopy = dsymm_itcopyTS, + .dtrmm_incopy = dtrmm_incopyTS, + .dtrmm_itcopy = dtrmm_itcopyTS, + .dtrmm_ounucopy = dtrmm_ounucopyTS, + .dtrmm_ounncopy = dtrmm_ounncopyTS, + .dtrmm_outucopy = dtrmm_outucopyTS, + .dtrmm_outncopy = dtrmm_outncopyTS, + .dtrmm_olnucopy = dtrmm_olnucopyTS, + .dtrmm_olnncopy = dtrmm_olnncopyTS, + .dtrmm_oltucopy = dtrmm_oltucopyTS, + .dtrmm_oltncopy = dtrmm_oltncopyTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - dsymm_iutcopyTS, dsymm_iltcopyTS, + .dsymm_iutcopy = dsymm_iutcopyTS, + .dsymm_iltcopy = dsymm_iltcopyTS, #else - dsymm_outcopyTS, dsymm_oltcopyTS, + .dsymm_iutcopy = dsymm_outcopyTS, + .dsymm_iltcopy = dsymm_oltcopyTS, #endif - dsymm_outcopyTS, dsymm_oltcopyTS, + .dsymm_outcopy = dsymm_outcopyTS, + .dsymm_oltcopy = dsymm_oltcopyTS, #ifndef NO_LAPACK - dneg_tcopyTS, dlaswp_ncopyTS, + .dneg_tcopy = dneg_tcopyTS, + .dlaswp_ncopy = dlaswp_ncopyTS, #else - NULL, NULL, + .dneg_tcopy = NULL, + .dlaswp_ncopy = NULL, #endif #endif #ifdef EXPRECISION - 0, 0, 0, - QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), - - qamax_kTS, qamin_kTS, qmax_kTS, qmin_kTS, - iqamax_kTS, iqamin_kTS, iqmax_kTS, iqmin_kTS, - qnrm2_kTS, qasum_kTS, qsum_kTS, qcopy_kTS, qdot_kTS, - qrot_kTS, qrotm_kTS, qaxpy_kTS, qscal_kTS, qswap_kTS, - qgemv_nTS, qgemv_tTS, qger_kTS, - qsymv_LTS, qsymv_UTS, - qgemm_kernelTS, qgemm_betaTS, + .qgemm_p = 0, + .qgemm_q = 0, + .qgemm_r = 0, + .qgemm_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .qgemm_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .qgemm_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), + + .qamax_k = qamax_kTS, + .qamin_k = qamin_kTS, + .qmax_k = qmax_kTS, + .qmin_k = qmin_kTS, + .iqamax_k = iqamax_kTS, + .iqamin_k = iqamin_kTS, + .iqmax_k = iqmax_kTS, + .iqmin_k = iqmin_kTS, + .qnrm2_k = qnrm2_kTS, + .qasum_k = qasum_kTS, + .qsum_k = qsum_kTS, + .qcopy_k = qcopy_kTS, + .qdot_k = qdot_kTS, + .qrot_k = qrot_kTS, + .qrotm_k = qrotm_kTS, + .qaxpy_k = qaxpy_kTS, + .qscal_k = qscal_kTS, + .qswap_k = qswap_kTS, + .qgemv_n = qgemv_nTS, + .qgemv_t = qgemv_tTS, + .qger_k = qger_kTS, + .qsymv_L = qsymv_LTS, + .qsymv_U = qsymv_UTS, + .qgemm_kernel = qgemm_kernelTS, + .qgemm_beta = qgemm_betaTS, #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qgemm_incopyTS, qgemm_itcopyTS, -#else - qgemm_oncopyTS, qgemm_otcopyTS, -#endif - qgemm_oncopyTS, qgemm_otcopyTS, - qtrsm_kernel_LNTS, qtrsm_kernel_LTTS, qtrsm_kernel_RNTS, qtrsm_kernel_RTTS, + .qgemm_incopy = qgemm_incopyTS, + .qgemm_itcopy = qgemm_itcopyTS, +#else + .qgemm_incopy = qgemm_oncopyTS, + .qgemm_itcopy = qgemm_otcopyTS, +#endif + .qgemm_oncopy = qgemm_oncopyTS, + .qgemm_otcopy = qgemm_otcopyTS, + .qtrsm_kernel_LN = qtrsm_kernel_LNTS, + .qtrsm_kernel_LT = qtrsm_kernel_LTTS, + .qtrsm_kernel_RN = qtrsm_kernel_RNTS, + .qtrsm_kernel_RT = qtrsm_kernel_RTTS, #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qtrsm_iunucopyTS, qtrsm_iunncopyTS, qtrsm_iutucopyTS, qtrsm_iutncopyTS, - qtrsm_ilnucopyTS, qtrsm_ilnncopyTS, qtrsm_iltucopyTS, qtrsm_iltncopyTS, -#else - qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS, - qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS, -#endif - qtrsm_ounucopyTS, qtrsm_ounncopyTS, qtrsm_outucopyTS, qtrsm_outncopyTS, - qtrsm_olnucopyTS, qtrsm_olnncopyTS, qtrsm_oltucopyTS, qtrsm_oltncopyTS, - qtrmm_kernel_RNTS, qtrmm_kernel_RTTS, qtrmm_kernel_LNTS, qtrmm_kernel_LTTS, + .qtrsm_iunucopy = qtrsm_iunucopyTS, + .qtrsm_iunncopy = qtrsm_iunncopyTS, + .qtrsm_iutucopy = qtrsm_iutucopyTS, + .qtrsm_iutncopy = qtrsm_iutncopyTS, + .qtrsm_ilnucopy = qtrsm_ilnucopyTS, + .qtrsm_ilnncopy = qtrsm_ilnncopyTS, + .qtrsm_iltucopy = qtrsm_iltucopyTS, + .qtrsm_iltncopy = qtrsm_iltncopyTS, +#else + .qtrsm_iunucopy = qtrsm_ounucopyTS, + .qtrsm_iunncopy = qtrsm_ounncopyTS, + .qtrsm_iutucopy = qtrsm_outucopyTS, + .qtrsm_iutncopy = qtrsm_outncopyTS, + .qtrsm_ilnucopy = qtrsm_olnucopyTS, + .qtrsm_ilnncopy = qtrsm_olnncopyTS, + .qtrsm_iltucopy = qtrsm_oltucopyTS, + .qtrsm_iltncopy = qtrsm_oltncopyTS, +#endif + .qtrsm_ounucopy = qtrsm_ounucopyTS, + .qtrsm_ounncopy = qtrsm_ounncopyTS, + .qtrsm_outucopy = qtrsm_outucopyTS, + .qtrsm_outncopy = qtrsm_outncopyTS, + .qtrsm_olnucopy = qtrsm_olnucopyTS, + .qtrsm_olnncopy = qtrsm_olnncopyTS, + .qtrsm_oltucopy = qtrsm_oltucopyTS, + .qtrsm_oltncopy = qtrsm_oltncopyTS, + .qtrmm_kernel_RN = qtrmm_kernel_RNTS, + .qtrmm_kernel_RT = qtrmm_kernel_RTTS, + .qtrmm_kernel_LN = qtrmm_kernel_LNTS, + .qtrmm_kernel_LT = qtrmm_kernel_LTTS, #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qtrmm_iunucopyTS, qtrmm_iunncopyTS, qtrmm_iutucopyTS, qtrmm_iutncopyTS, - qtrmm_ilnucopyTS, qtrmm_ilnncopyTS, qtrmm_iltucopyTS, qtrmm_iltncopyTS, -#else - qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS, - qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS, -#endif - qtrmm_ounucopyTS, qtrmm_ounncopyTS, qtrmm_outucopyTS, qtrmm_outncopyTS, - qtrmm_olnucopyTS, qtrmm_olnncopyTS, qtrmm_oltucopyTS, qtrmm_oltncopyTS, + .qtrmm_iunucopy = qtrmm_iunucopyTS, + .qtrmm_iunncopy = qtrmm_iunncopyTS, + .qtrmm_iutucopy = qtrmm_iutucopyTS, + .qtrmm_iutncopy = qtrmm_iutncopyTS, + .qtrmm_ilnucopy = qtrmm_ilnucopyTS, + .qtrmm_ilnncopy = qtrmm_ilnncopyTS, + .qtrmm_iltucopy = qtrmm_iltucopyTS, + .qtrmm_iltncopy = qtrmm_iltncopyTS, +#else + .qtrmm_iunucopy = qtrmm_ounucopyTS, + .qtrmm_iunncopy = qtrmm_ounncopyTS, + .qtrmm_iutucopy = qtrmm_outucopyTS, + .qtrmm_iutncopy = qtrmm_outncopyTS, + .qtrmm_ilnucopy = qtrmm_olnucopyTS, + .qtrmm_ilnncopy = qtrmm_olnncopyTS, + .qtrmm_iltucopy = qtrmm_oltucopyTS, + .qtrmm_iltncopy = qtrmm_oltncopyTS, +#endif + .qtrmm_ounucopy = qtrmm_ounucopyTS, + .qtrmm_ounncopy = qtrmm_ounncopyTS, + .qtrmm_outucopy = qtrmm_outucopyTS, + .qtrmm_outncopy = qtrmm_outncopyTS, + .qtrmm_olnucopy = qtrmm_olnucopyTS, + .qtrmm_olnncopy = qtrmm_olnncopyTS, + .qtrmm_oltucopy = qtrmm_oltucopyTS, + .qtrmm_oltncopy = qtrmm_oltncopyTS, #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N - qsymm_iutcopyTS, qsymm_iltcopyTS, + .qsymm_iutcopy = qsymm_iutcopyTS, + .qsymm_iltcopy = qsymm_iltcopyTS, #else - qsymm_outcopyTS, qsymm_oltcopyTS, + .qsymm_iutcopy = qsymm_outcopyTS, + .qsymm_iltcopy = qsymm_oltcopyTS, #endif - qsymm_outcopyTS, qsymm_oltcopyTS, + .qsymm_outcopy = qsymm_outcopyTS, + .qsymm_oltcopy = qsymm_oltcopyTS, #ifndef NO_LAPACK - qneg_tcopyTS, qlaswp_ncopyTS, + .qneg_tcopy = qneg_tcopyTS, + .qlaswp_ncopy = qlaswp_ncopyTS, #else - NULL, NULL, + .qneg_tcopy = NULL, + .qlaswp_ncopy = NULL, #endif #endif #if (BUILD_COMPLEX) - 0, 0, 0, - CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N, + .cgemm_p = 0, + .cgemm_q = 0, + .cgemm_r = 0, + .cgemm_unroll_m = CGEMM_DEFAULT_UNROLL_M, + .cgemm_unroll_n = CGEMM_DEFAULT_UNROLL_N, #ifdef CGEMM_DEFAULT_UNROLL_MN - CGEMM_DEFAULT_UNROLL_MN, + .cgemm_unroll_mn = CGEMM_DEFAULT_UNROLL_MN, #else - MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), + .cgemm_unroll_mn = MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), #endif #if (BUILD_COMPLEX) - camax_kTS, camin_kTS, + .camax_k = camax_kTS, + .camin_k = camin_kTS, #endif #if (BUILD_COMPLEX) - icamax_kTS, + .icamax_k = icamax_kTS, #endif #if (BUILD_COMPLEX) - icamin_kTS, - cnrm2_kTS, casum_kTS, csum_kTS, + .icamin_k = icamin_kTS, + .cnrm2_k = cnrm2_kTS, + .casum_k = casum_kTS, + .csum_k = csum_kTS, #endif #if (BUILD_COMPLEX) - ccopy_kTS, cdotu_kTS, cdotc_kTS, + .ccopy_k = ccopy_kTS, + .cdotu_k = cdotu_kTS, + .cdotc_k = cdotc_kTS, #endif #if (BUILD_COMPLEX) - csrot_kTS, + .csrot_k = csrot_kTS, #endif #if (BUILD_COMPLEX) - caxpy_kTS, - caxpyc_kTS, - cscal_kTS, - cswap_kTS, - - cgemv_nTS, cgemv_tTS, cgemv_rTS, cgemv_cTS, - cgemv_oTS, cgemv_uTS, cgemv_sTS, cgemv_dTS, + .caxpy_k = caxpy_kTS, + .caxpyc_k = caxpyc_kTS, + .cscal_k = cscal_kTS, + .cswap_k = cswap_kTS, + + .cgemv_n = cgemv_nTS, + .cgemv_t = cgemv_tTS, + .cgemv_r = cgemv_rTS, + .cgemv_c = cgemv_cTS, + .cgemv_o = cgemv_oTS, + .cgemv_u = cgemv_uTS, + .cgemv_s = cgemv_sTS, + .cgemv_d = cgemv_dTS, #endif #if (BUILD_COMPLEX) - cgeru_kTS, cgerc_kTS, cgerv_kTS, cgerd_kTS, - csymv_LTS, csymv_UTS, - chemv_LTS, chemv_UTS, chemv_MTS, chemv_VTS, + .cgeru_k = cgeru_kTS, + .cgerc_k = cgerc_kTS, + .cgerv_k = cgerv_kTS, + .cgerd_k = cgerd_kTS, + .csymv_L = csymv_LTS, + .csymv_U = csymv_UTS, + .chemv_L = chemv_LTS, + .chemv_U = chemv_UTS, + .chemv_M = chemv_MTS, + .chemv_V = chemv_VTS, #endif #if (BUILD_COMPLEX) #ifdef ARCH_ARM64 #ifdef HAVE_SME - sme_cgemm_kernelTS, -#else - NULL, -#endif -#endif - cgemm_kernel_nTS, cgemm_kernel_lTS, cgemm_kernel_rTS, cgemm_kernel_bTS, - cgemm_betaTS, - csymm_kernel_nTS, csymm_kernel_lTS, csymm_kernel_rTS, csymm_kernel_bTS, - ctrmm_gemm_kernel_nTS, ctrmm_gemm_kernel_lTS, ctrmm_gemm_kernel_rTS, ctrmm_gemm_kernel_bTS, + .sme_cgemm_kernel = sme_cgemm_kernelTS, +#else + .sme_cgemm_kernel = NULL, +#endif +#endif + .cgemm_kernel_n = cgemm_kernel_nTS, + .cgemm_kernel_l = cgemm_kernel_lTS, + .cgemm_kernel_r = cgemm_kernel_rTS, + .cgemm_kernel_b = cgemm_kernel_bTS, + .cgemm_beta = cgemm_betaTS, + .csymm_kernel_n = csymm_kernel_nTS, + .csymm_kernel_l = csymm_kernel_lTS, + .csymm_kernel_r = csymm_kernel_rTS, + .csymm_kernel_b = csymm_kernel_bTS, + .ctrmm_gemm_kernel_n = ctrmm_gemm_kernel_nTS, + .ctrmm_gemm_kernel_l = ctrmm_gemm_kernel_lTS, + .ctrmm_gemm_kernel_r = ctrmm_gemm_kernel_rTS, + .ctrmm_gemm_kernel_b = ctrmm_gemm_kernel_bTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - cgemm_incopyTS, cgemm_itcopyTS, + .cgemm_incopy = cgemm_incopyTS, + .cgemm_itcopy = cgemm_itcopyTS, #else - cgemm_oncopyTS, cgemm_otcopyTS, + .cgemm_incopy = cgemm_oncopyTS, + .cgemm_itcopy = cgemm_otcopyTS, #endif - cgemm_oncopyTS, cgemm_otcopyTS, + .cgemm_oncopy = cgemm_oncopyTS, + .cgemm_otcopy = cgemm_otcopyTS, #ifdef SMALL_MATRIX_OPT - cgemm_small_matrix_permitTS, - cgemm_small_kernel_nnTS, cgemm_small_kernel_ntTS, cgemm_small_kernel_nrTS, cgemm_small_kernel_ncTS, - cgemm_small_kernel_tnTS, cgemm_small_kernel_ttTS, cgemm_small_kernel_trTS, cgemm_small_kernel_tcTS, - cgemm_small_kernel_rnTS, cgemm_small_kernel_rtTS, cgemm_small_kernel_rrTS, cgemm_small_kernel_rcTS, - cgemm_small_kernel_cnTS, cgemm_small_kernel_ctTS, cgemm_small_kernel_crTS, cgemm_small_kernel_ccTS, - cgemm_small_kernel_b0_nnTS, cgemm_small_kernel_b0_ntTS, cgemm_small_kernel_b0_nrTS, cgemm_small_kernel_b0_ncTS, - cgemm_small_kernel_b0_tnTS, cgemm_small_kernel_b0_ttTS, cgemm_small_kernel_b0_trTS, cgemm_small_kernel_b0_tcTS, - cgemm_small_kernel_b0_rnTS, cgemm_small_kernel_b0_rtTS, cgemm_small_kernel_b0_rrTS, cgemm_small_kernel_b0_rcTS, - cgemm_small_kernel_b0_cnTS, cgemm_small_kernel_b0_ctTS, cgemm_small_kernel_b0_crTS, cgemm_small_kernel_b0_ccTS, -#endif - - ctrsm_kernel_LNTS, ctrsm_kernel_LTTS, ctrsm_kernel_LRTS, ctrsm_kernel_LCTS, - ctrsm_kernel_RNTS, ctrsm_kernel_RTTS, ctrsm_kernel_RRTS, ctrsm_kernel_RCTS, + .cgemm_small_matrix_permit = cgemm_small_matrix_permitTS, + .cgemm_small_kernel_nn = cgemm_small_kernel_nnTS, + .cgemm_small_kernel_nt = cgemm_small_kernel_ntTS, + .cgemm_small_kernel_nr = cgemm_small_kernel_nrTS, + .cgemm_small_kernel_nc = cgemm_small_kernel_ncTS, + .cgemm_small_kernel_tn = cgemm_small_kernel_tnTS, + .cgemm_small_kernel_tt = cgemm_small_kernel_ttTS, + .cgemm_small_kernel_tr = cgemm_small_kernel_trTS, + .cgemm_small_kernel_tc = cgemm_small_kernel_tcTS, + .cgemm_small_kernel_rn = cgemm_small_kernel_rnTS, + .cgemm_small_kernel_rt = cgemm_small_kernel_rtTS, + .cgemm_small_kernel_rr = cgemm_small_kernel_rrTS, + .cgemm_small_kernel_rc = cgemm_small_kernel_rcTS, + .cgemm_small_kernel_cn = cgemm_small_kernel_cnTS, + .cgemm_small_kernel_ct = cgemm_small_kernel_ctTS, + .cgemm_small_kernel_cr = cgemm_small_kernel_crTS, + .cgemm_small_kernel_cc = cgemm_small_kernel_ccTS, + .cgemm_small_kernel_b0_nn = cgemm_small_kernel_b0_nnTS, + .cgemm_small_kernel_b0_nt = cgemm_small_kernel_b0_ntTS, + .cgemm_small_kernel_b0_nr = cgemm_small_kernel_b0_nrTS, + .cgemm_small_kernel_b0_nc = cgemm_small_kernel_b0_ncTS, + .cgemm_small_kernel_b0_tn = cgemm_small_kernel_b0_tnTS, + .cgemm_small_kernel_b0_tt = cgemm_small_kernel_b0_ttTS, + .cgemm_small_kernel_b0_tr = cgemm_small_kernel_b0_trTS, + .cgemm_small_kernel_b0_tc = cgemm_small_kernel_b0_tcTS, + .cgemm_small_kernel_b0_rn = cgemm_small_kernel_b0_rnTS, + .cgemm_small_kernel_b0_rt = cgemm_small_kernel_b0_rtTS, + .cgemm_small_kernel_b0_rr = cgemm_small_kernel_b0_rrTS, + .cgemm_small_kernel_b0_rc = cgemm_small_kernel_b0_rcTS, + .cgemm_small_kernel_b0_cn = cgemm_small_kernel_b0_cnTS, + .cgemm_small_kernel_b0_ct = cgemm_small_kernel_b0_ctTS, + .cgemm_small_kernel_b0_cr = cgemm_small_kernel_b0_crTS, + .cgemm_small_kernel_b0_cc = cgemm_small_kernel_b0_ccTS, +#endif + + .ctrsm_kernel_LN = ctrsm_kernel_LNTS, + .ctrsm_kernel_LT = ctrsm_kernel_LTTS, + .ctrsm_kernel_LR = ctrsm_kernel_LRTS, + .ctrsm_kernel_LC = ctrsm_kernel_LCTS, + .ctrsm_kernel_RN = ctrsm_kernel_RNTS, + .ctrsm_kernel_RT = ctrsm_kernel_RTTS, + .ctrsm_kernel_RR = ctrsm_kernel_RRTS, + .ctrsm_kernel_RC = ctrsm_kernel_RCTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - ctrsm_iunucopyTS, ctrsm_iunncopyTS, ctrsm_iutucopyTS, ctrsm_iutncopyTS, - ctrsm_ilnucopyTS, ctrsm_ilnncopyTS, ctrsm_iltucopyTS, ctrsm_iltncopyTS, -#else - ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS, - ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS, -#endif - ctrsm_ounucopyTS, ctrsm_ounncopyTS, ctrsm_outucopyTS, ctrsm_outncopyTS, - ctrsm_olnucopyTS, ctrsm_olnncopyTS, ctrsm_oltucopyTS, ctrsm_oltncopyTS, + .ctrsm_iunucopy = ctrsm_iunucopyTS, + .ctrsm_iunncopy = ctrsm_iunncopyTS, + .ctrsm_iutucopy = ctrsm_iutucopyTS, + .ctrsm_iutncopy = ctrsm_iutncopyTS, + .ctrsm_ilnucopy = ctrsm_ilnucopyTS, + .ctrsm_ilnncopy = ctrsm_ilnncopyTS, + .ctrsm_iltucopy = ctrsm_iltucopyTS, + .ctrsm_iltncopy = ctrsm_iltncopyTS, +#else + .ctrsm_iunucopy = ctrsm_ounucopyTS, + .ctrsm_iunncopy = ctrsm_ounncopyTS, + .ctrsm_iutucopy = ctrsm_outucopyTS, + .ctrsm_iutncopy = ctrsm_outncopyTS, + .ctrsm_ilnucopy = ctrsm_olnucopyTS, + .ctrsm_ilnncopy = ctrsm_olnncopyTS, + .ctrsm_iltucopy = ctrsm_oltucopyTS, + .ctrsm_iltncopy = ctrsm_oltncopyTS, +#endif + .ctrsm_ounucopy = ctrsm_ounucopyTS, + .ctrsm_ounncopy = ctrsm_ounncopyTS, + .ctrsm_outucopy = ctrsm_outucopyTS, + .ctrsm_outncopy = ctrsm_outncopyTS, + .ctrsm_olnucopy = ctrsm_olnucopyTS, + .ctrsm_olnncopy = ctrsm_olnncopyTS, + .ctrsm_oltucopy = ctrsm_oltucopyTS, + .ctrsm_oltncopy = ctrsm_oltncopyTS, #endif #endif #if (BUILD_COMPLEX) - ctrmm_kernel_RNTS, ctrmm_kernel_RTTS, ctrmm_kernel_RRTS, ctrmm_kernel_RCTS, - ctrmm_kernel_LNTS, ctrmm_kernel_LTTS, ctrmm_kernel_LRTS, ctrmm_kernel_LCTS, + .ctrmm_kernel_RN = ctrmm_kernel_RNTS, + .ctrmm_kernel_RT = ctrmm_kernel_RTTS, + .ctrmm_kernel_RR = ctrmm_kernel_RRTS, + .ctrmm_kernel_RC = ctrmm_kernel_RCTS, + .ctrmm_kernel_LN = ctrmm_kernel_LNTS, + .ctrmm_kernel_LT = ctrmm_kernel_LTTS, + .ctrmm_kernel_LR = ctrmm_kernel_LRTS, + .ctrmm_kernel_LC = ctrmm_kernel_LCTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - ctrmm_iunucopyTS, ctrmm_iunncopyTS, ctrmm_iutucopyTS, ctrmm_iutncopyTS, - ctrmm_ilnucopyTS, ctrmm_ilnncopyTS, ctrmm_iltucopyTS, ctrmm_iltncopyTS, -#else - ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, - ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, -#endif - csymm_incopyTS, csymm_itcopyTS, - ctrmm_incopyTS, ctrmm_itcopyTS, - ctrmm_ounucopyTS, ctrmm_ounncopyTS, ctrmm_outucopyTS, ctrmm_outncopyTS, - ctrmm_olnucopyTS, ctrmm_olnncopyTS, ctrmm_oltucopyTS, ctrmm_oltncopyTS, + .ctrmm_iunucopy = ctrmm_iunucopyTS, + .ctrmm_iunncopy = ctrmm_iunncopyTS, + .ctrmm_iutucopy = ctrmm_iutucopyTS, + .ctrmm_iutncopy = ctrmm_iutncopyTS, + .ctrmm_ilnucopy = ctrmm_ilnucopyTS, + .ctrmm_ilnncopy = ctrmm_ilnncopyTS, + .ctrmm_iltucopy = ctrmm_iltucopyTS, + .ctrmm_iltncopy = ctrmm_iltncopyTS, +#else + .ctrmm_iunucopy = ctrmm_ounucopyTS, + .ctrmm_iunncopy = ctrmm_ounncopyTS, + .ctrmm_iutucopy = ctrmm_outucopyTS, + .ctrmm_iutncopy = ctrmm_outncopyTS, + .ctrmm_ilnucopy = ctrmm_olnucopyTS, + .ctrmm_ilnncopy = ctrmm_olnncopyTS, + .ctrmm_iltucopy = ctrmm_oltucopyTS, + .ctrmm_iltncopy = ctrmm_oltncopyTS, +#endif + .csymm_incopy = csymm_incopyTS, + .csymm_itcopy = csymm_itcopyTS, + .ctrmm_incopy = ctrmm_incopyTS, + .ctrmm_itcopy = ctrmm_itcopyTS, + .ctrmm_ounucopy = ctrmm_ounucopyTS, + .ctrmm_ounncopy = ctrmm_ounncopyTS, + .ctrmm_outucopy = ctrmm_outucopyTS, + .ctrmm_outncopy = ctrmm_outncopyTS, + .ctrmm_olnucopy = ctrmm_olnucopyTS, + .ctrmm_olnncopy = ctrmm_olnncopyTS, + .ctrmm_oltucopy = ctrmm_oltucopyTS, + .ctrmm_oltncopy = ctrmm_oltncopyTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - csymm_iutcopyTS, csymm_iltcopyTS, + .csymm_iutcopy = csymm_iutcopyTS, + .csymm_iltcopy = csymm_iltcopyTS, #else - csymm_outcopyTS, csymm_oltcopyTS, + .csymm_iutcopy = csymm_outcopyTS, + .csymm_iltcopy = csymm_oltcopyTS, #endif - csymm_outcopyTS, csymm_oltcopyTS, + .csymm_outcopy = csymm_outcopyTS, + .csymm_oltcopy = csymm_oltcopyTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - chemm_iutcopyTS, chemm_iltcopyTS, + .chemm_iutcopy = chemm_iutcopyTS, + .chemm_iltcopy = chemm_iltcopyTS, #else - chemm_outcopyTS, chemm_oltcopyTS, + .chemm_iutcopy = chemm_outcopyTS, + .chemm_iltcopy = chemm_oltcopyTS, #endif - chemm_outcopyTS, chemm_oltcopyTS, + .chemm_outcopy = chemm_outcopyTS, + .chemm_oltcopy = chemm_oltcopyTS, - 0, 0, 0, + .cgemm3m_p = 0, + .cgemm3m_q = 0, + .cgemm3m_r = 0, #if (USE_GEMM3M) #ifdef CGEMM3M_DEFAULT_UNROLL_M - CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N, MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), -#else - SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N, MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), -#endif - - - cgemm3m_kernelTS, - - cgemm3m_incopybTS, cgemm3m_incopyrTS, - cgemm3m_incopyiTS, cgemm3m_itcopybTS, - cgemm3m_itcopyrTS, cgemm3m_itcopyiTS, - cgemm3m_oncopybTS, cgemm3m_oncopyrTS, - cgemm3m_oncopyiTS, cgemm3m_otcopybTS, - cgemm3m_otcopyrTS, cgemm3m_otcopyiTS, - - csymm3m_iucopybTS, csymm3m_ilcopybTS, - csymm3m_iucopyrTS, csymm3m_ilcopyrTS, - csymm3m_iucopyiTS, csymm3m_ilcopyiTS, - csymm3m_oucopybTS, csymm3m_olcopybTS, - csymm3m_oucopyrTS, csymm3m_olcopyrTS, - csymm3m_oucopyiTS, csymm3m_olcopyiTS, - - chemm3m_iucopybTS, chemm3m_ilcopybTS, - chemm3m_iucopyrTS, chemm3m_ilcopyrTS, - chemm3m_iucopyiTS, chemm3m_ilcopyiTS, - - chemm3m_oucopybTS, chemm3m_olcopybTS, - chemm3m_oucopyrTS, chemm3m_olcopyrTS, - chemm3m_oucopyiTS, chemm3m_olcopyiTS, -#else - 0, 0, 0, - - NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, + .cgemm3m_unroll_m = CGEMM3M_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = CGEMM3M_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), +#else + .cgemm3m_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = SGEMM_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#endif + + + .cgemm3m_kernel = cgemm3m_kernelTS, + + .cgemm3m_incopyb = cgemm3m_incopybTS, + .cgemm3m_incopyr = cgemm3m_incopyrTS, + .cgemm3m_incopyi = cgemm3m_incopyiTS, + .cgemm3m_itcopyb = cgemm3m_itcopybTS, + .cgemm3m_itcopyr = cgemm3m_itcopyrTS, + .cgemm3m_itcopyi = cgemm3m_itcopyiTS, + .cgemm3m_oncopyb = cgemm3m_oncopybTS, + .cgemm3m_oncopyr = cgemm3m_oncopyrTS, + .cgemm3m_oncopyi = cgemm3m_oncopyiTS, + .cgemm3m_otcopyb = cgemm3m_otcopybTS, + .cgemm3m_otcopyr = cgemm3m_otcopyrTS, + .cgemm3m_otcopyi = cgemm3m_otcopyiTS, + + .csymm3m_iucopyb = csymm3m_iucopybTS, + .csymm3m_ilcopyb = csymm3m_ilcopybTS, + .csymm3m_iucopyr = csymm3m_iucopyrTS, + .csymm3m_ilcopyr = csymm3m_ilcopyrTS, + .csymm3m_iucopyi = csymm3m_iucopyiTS, + .csymm3m_ilcopyi = csymm3m_ilcopyiTS, + .csymm3m_oucopyb = csymm3m_oucopybTS, + .csymm3m_olcopyb = csymm3m_olcopybTS, + .csymm3m_oucopyr = csymm3m_oucopyrTS, + .csymm3m_olcopyr = csymm3m_olcopyrTS, + .csymm3m_oucopyi = csymm3m_oucopyiTS, + .csymm3m_olcopyi = csymm3m_olcopyiTS, + + .chemm3m_iucopyb = chemm3m_iucopybTS, + .chemm3m_ilcopyb = chemm3m_ilcopybTS, + .chemm3m_iucopyr = chemm3m_iucopyrTS, + .chemm3m_ilcopyr = chemm3m_ilcopyrTS, + .chemm3m_iucopyi = chemm3m_iucopyiTS, + .chemm3m_ilcopyi = chemm3m_ilcopyiTS, + + .chemm3m_oucopyb = chemm3m_oucopybTS, + .chemm3m_olcopyb = chemm3m_olcopybTS, + .chemm3m_oucopyr = chemm3m_oucopyrTS, + .chemm3m_olcopyr = chemm3m_olcopyrTS, + .chemm3m_oucopyi = chemm3m_oucopyiTS, + .chemm3m_olcopyi = chemm3m_olcopyiTS, +#else + .cgemm3m_unroll_m = 0, + .cgemm3m_unroll_n = 0, + .cgemm3m_unroll_mn = 0, + + .cgemm3m_kernel = NULL, + + .cgemm3m_incopyb = NULL, + .cgemm3m_incopyr = NULL, + .cgemm3m_incopyi = NULL, + .cgemm3m_itcopyb = NULL, + .cgemm3m_itcopyr = NULL, + .cgemm3m_itcopyi = NULL, + .cgemm3m_oncopyb = NULL, + .cgemm3m_oncopyr = NULL, + .cgemm3m_oncopyi = NULL, + .cgemm3m_otcopyb = NULL, + .cgemm3m_otcopyr = NULL, + .cgemm3m_otcopyi = NULL, + + .csymm3m_iucopyb = NULL, + .csymm3m_ilcopyb = NULL, + .csymm3m_iucopyr = NULL, + .csymm3m_ilcopyr = NULL, + .csymm3m_iucopyi = NULL, + .csymm3m_ilcopyi = NULL, + .csymm3m_oucopyb = NULL, + .csymm3m_olcopyb = NULL, + .csymm3m_oucopyr = NULL, + .csymm3m_olcopyr = NULL, + .csymm3m_oucopyi = NULL, + .csymm3m_olcopyi = NULL, + + .chemm3m_iucopyb = NULL, + .chemm3m_ilcopyb = NULL, + .chemm3m_iucopyr = NULL, + .chemm3m_ilcopyr = NULL, + .chemm3m_iucopyi = NULL, + .chemm3m_ilcopyi = NULL, + + .chemm3m_oucopyb = NULL, + .chemm3m_olcopyb = NULL, + .chemm3m_oucopyr = NULL, + .chemm3m_olcopyr = NULL, + .chemm3m_oucopyi = NULL, + .chemm3m_olcopyi = NULL, #endif #endif #if (BUILD_COMPLEX) #ifndef NO_LAPACK - cneg_tcopyTS, + .cneg_tcopy = cneg_tcopyTS, - claswp_ncopyTS, + .claswp_ncopy = claswp_ncopyTS, #else - NULL, NULL, + .cneg_tcopy = NULL, + .claswp_ncopy = NULL, #endif #endif #if BUILD_COMPLEX16 == 1 - 0, 0, 0, - ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N, + .zgemm_p = 0, + .zgemm_q = 0, + .zgemm_r = 0, + .zgemm_unroll_m = ZGEMM_DEFAULT_UNROLL_M, + .zgemm_unroll_n = ZGEMM_DEFAULT_UNROLL_N, #ifdef ZGEMM_DEFAULT_UNROLL_MN - ZGEMM_DEFAULT_UNROLL_MN, -#else - MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), -#endif - - zamax_kTS, zamin_kTS, izamax_kTS, izamin_kTS, - znrm2_kTS, zasum_kTS, zsum_kTS, zcopy_kTS, - zdotu_kTS, zdotc_kTS, zdrot_kTS, - zaxpy_kTS, zaxpyc_kTS, zscal_kTS, zswap_kTS, - - zgemv_nTS, zgemv_tTS, zgemv_rTS, zgemv_cTS, - zgemv_oTS, zgemv_uTS, zgemv_sTS, zgemv_dTS, - zgeru_kTS, zgerc_kTS, zgerv_kTS, zgerd_kTS, - zsymv_LTS, zsymv_UTS, - zhemv_LTS, zhemv_UTS, zhemv_MTS, zhemv_VTS, + .zgemm_unroll_mn = ZGEMM_DEFAULT_UNROLL_MN, +#else + .zgemm_unroll_mn = MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), +#endif + + .zamax_k = zamax_kTS, + .zamin_k = zamin_kTS, + .izamax_k = izamax_kTS, + .izamin_k = izamin_kTS, + .znrm2_k = znrm2_kTS, + .zasum_k = zasum_kTS, + .zsum_k = zsum_kTS, + .zcopy_k = zcopy_kTS, + .zdotu_k = zdotu_kTS, + .zdotc_k = zdotc_kTS, + .zdrot_k = zdrot_kTS, + .zaxpy_k = zaxpy_kTS, + .zaxpyc_k = zaxpyc_kTS, + .zscal_k = zscal_kTS, + .zswap_k = zswap_kTS, + + .zgemv_n = zgemv_nTS, + .zgemv_t = zgemv_tTS, + .zgemv_r = zgemv_rTS, + .zgemv_c = zgemv_cTS, + .zgemv_o = zgemv_oTS, + .zgemv_u = zgemv_uTS, + .zgemv_s = zgemv_sTS, + .zgemv_d = zgemv_dTS, + .zgeru_k = zgeru_kTS, + .zgerc_k = zgerc_kTS, + .zgerv_k = zgerv_kTS, + .zgerd_k = zgerd_kTS, + .zsymv_L = zsymv_LTS, + .zsymv_U = zsymv_UTS, + .zhemv_L = zhemv_LTS, + .zhemv_U = zhemv_UTS, + .zhemv_M = zhemv_MTS, + .zhemv_V = zhemv_VTS, #ifdef ARCH_ARM64 #ifdef HAVE_SME - sme_zgemm_kernelTS, + .sme_zgemm_kernel = sme_zgemm_kernelTS, #else - NULL, + .sme_zgemm_kernel = NULL, #endif #endif - zgemm_kernel_nTS, zgemm_kernel_lTS, zgemm_kernel_rTS, zgemm_kernel_bTS, - zgemm_betaTS, - zsymm_kernel_nTS, zsymm_kernel_lTS, zsymm_kernel_rTS, zsymm_kernel_bTS, - ztrmm_gemm_kernel_nTS, ztrmm_gemm_kernel_lTS, ztrmm_gemm_kernel_rTS, ztrmm_gemm_kernel_bTS, + .zgemm_kernel_n = zgemm_kernel_nTS, + .zgemm_kernel_l = zgemm_kernel_lTS, + .zgemm_kernel_r = zgemm_kernel_rTS, + .zgemm_kernel_b = zgemm_kernel_bTS, + .zgemm_beta = zgemm_betaTS, + .zsymm_kernel_n = zsymm_kernel_nTS, + .zsymm_kernel_l = zsymm_kernel_lTS, + .zsymm_kernel_r = zsymm_kernel_rTS, + .zsymm_kernel_b = zsymm_kernel_bTS, + .ztrmm_gemm_kernel_n = ztrmm_gemm_kernel_nTS, + .ztrmm_gemm_kernel_l = ztrmm_gemm_kernel_lTS, + .ztrmm_gemm_kernel_r = ztrmm_gemm_kernel_rTS, + .ztrmm_gemm_kernel_b = ztrmm_gemm_kernel_bTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zgemm_incopyTS, zgemm_itcopyTS, + .zgemm_incopy = zgemm_incopyTS, + .zgemm_itcopy = zgemm_itcopyTS, #else - zgemm_oncopyTS, zgemm_otcopyTS, + .zgemm_incopy = zgemm_oncopyTS, + .zgemm_itcopy = zgemm_otcopyTS, #endif - zgemm_oncopyTS, zgemm_otcopyTS, + .zgemm_oncopy = zgemm_oncopyTS, + .zgemm_otcopy = zgemm_otcopyTS, #ifdef SMALL_MATRIX_OPT - zgemm_small_matrix_permitTS, - zgemm_small_kernel_nnTS, zgemm_small_kernel_ntTS, zgemm_small_kernel_nrTS, zgemm_small_kernel_ncTS, - zgemm_small_kernel_tnTS, zgemm_small_kernel_ttTS, zgemm_small_kernel_trTS, zgemm_small_kernel_tcTS, - zgemm_small_kernel_rnTS, zgemm_small_kernel_rtTS, zgemm_small_kernel_rrTS, zgemm_small_kernel_rcTS, - zgemm_small_kernel_cnTS, zgemm_small_kernel_ctTS, zgemm_small_kernel_crTS, zgemm_small_kernel_ccTS, - zgemm_small_kernel_b0_nnTS, zgemm_small_kernel_b0_ntTS, zgemm_small_kernel_b0_nrTS, zgemm_small_kernel_b0_ncTS, - zgemm_small_kernel_b0_tnTS, zgemm_small_kernel_b0_ttTS, zgemm_small_kernel_b0_trTS, zgemm_small_kernel_b0_tcTS, - zgemm_small_kernel_b0_rnTS, zgemm_small_kernel_b0_rtTS, zgemm_small_kernel_b0_rrTS, zgemm_small_kernel_b0_rcTS, - zgemm_small_kernel_b0_cnTS, zgemm_small_kernel_b0_ctTS, zgemm_small_kernel_b0_crTS, zgemm_small_kernel_b0_ccTS, -#endif - - ztrsm_kernel_LNTS, ztrsm_kernel_LTTS, ztrsm_kernel_LRTS, ztrsm_kernel_LCTS, - ztrsm_kernel_RNTS, ztrsm_kernel_RTTS, ztrsm_kernel_RRTS, ztrsm_kernel_RCTS, + .zgemm_small_matrix_permit = zgemm_small_matrix_permitTS, + .zgemm_small_kernel_nn = zgemm_small_kernel_nnTS, + .zgemm_small_kernel_nt = zgemm_small_kernel_ntTS, + .zgemm_small_kernel_nr = zgemm_small_kernel_nrTS, + .zgemm_small_kernel_nc = zgemm_small_kernel_ncTS, + .zgemm_small_kernel_tn = zgemm_small_kernel_tnTS, + .zgemm_small_kernel_tt = zgemm_small_kernel_ttTS, + .zgemm_small_kernel_tr = zgemm_small_kernel_trTS, + .zgemm_small_kernel_tc = zgemm_small_kernel_tcTS, + .zgemm_small_kernel_rn = zgemm_small_kernel_rnTS, + .zgemm_small_kernel_rt = zgemm_small_kernel_rtTS, + .zgemm_small_kernel_rr = zgemm_small_kernel_rrTS, + .zgemm_small_kernel_rc = zgemm_small_kernel_rcTS, + .zgemm_small_kernel_cn = zgemm_small_kernel_cnTS, + .zgemm_small_kernel_ct = zgemm_small_kernel_ctTS, + .zgemm_small_kernel_cr = zgemm_small_kernel_crTS, + .zgemm_small_kernel_cc = zgemm_small_kernel_ccTS, + .zgemm_small_kernel_b0_nn = zgemm_small_kernel_b0_nnTS, + .zgemm_small_kernel_b0_nt = zgemm_small_kernel_b0_ntTS, + .zgemm_small_kernel_b0_nr = zgemm_small_kernel_b0_nrTS, + .zgemm_small_kernel_b0_nc = zgemm_small_kernel_b0_ncTS, + .zgemm_small_kernel_b0_tn = zgemm_small_kernel_b0_tnTS, + .zgemm_small_kernel_b0_tt = zgemm_small_kernel_b0_ttTS, + .zgemm_small_kernel_b0_tr = zgemm_small_kernel_b0_trTS, + .zgemm_small_kernel_b0_tc = zgemm_small_kernel_b0_tcTS, + .zgemm_small_kernel_b0_rn = zgemm_small_kernel_b0_rnTS, + .zgemm_small_kernel_b0_rt = zgemm_small_kernel_b0_rtTS, + .zgemm_small_kernel_b0_rr = zgemm_small_kernel_b0_rrTS, + .zgemm_small_kernel_b0_rc = zgemm_small_kernel_b0_rcTS, + .zgemm_small_kernel_b0_cn = zgemm_small_kernel_b0_cnTS, + .zgemm_small_kernel_b0_ct = zgemm_small_kernel_b0_ctTS, + .zgemm_small_kernel_b0_cr = zgemm_small_kernel_b0_crTS, + .zgemm_small_kernel_b0_cc = zgemm_small_kernel_b0_ccTS, +#endif + + .ztrsm_kernel_LN = ztrsm_kernel_LNTS, + .ztrsm_kernel_LT = ztrsm_kernel_LTTS, + .ztrsm_kernel_LR = ztrsm_kernel_LRTS, + .ztrsm_kernel_LC = ztrsm_kernel_LCTS, + .ztrsm_kernel_RN = ztrsm_kernel_RNTS, + .ztrsm_kernel_RT = ztrsm_kernel_RTTS, + .ztrsm_kernel_RR = ztrsm_kernel_RRTS, + .ztrsm_kernel_RC = ztrsm_kernel_RCTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - ztrsm_iunucopyTS, ztrsm_iunncopyTS, ztrsm_iutucopyTS, ztrsm_iutncopyTS, - ztrsm_ilnucopyTS, ztrsm_ilnncopyTS, ztrsm_iltucopyTS, ztrsm_iltncopyTS, -#else - ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS, - ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS, -#endif - ztrsm_ounucopyTS, ztrsm_ounncopyTS, ztrsm_outucopyTS, ztrsm_outncopyTS, - ztrsm_olnucopyTS, ztrsm_olnncopyTS, ztrsm_oltucopyTS, ztrsm_oltncopyTS, - - ztrmm_kernel_RNTS, ztrmm_kernel_RTTS, ztrmm_kernel_RRTS, ztrmm_kernel_RCTS, - ztrmm_kernel_LNTS, ztrmm_kernel_LTTS, ztrmm_kernel_LRTS, ztrmm_kernel_LCTS, + .ztrsm_iunucopy = ztrsm_iunucopyTS, + .ztrsm_iunncopy = ztrsm_iunncopyTS, + .ztrsm_iutucopy = ztrsm_iutucopyTS, + .ztrsm_iutncopy = ztrsm_iutncopyTS, + .ztrsm_ilnucopy = ztrsm_ilnucopyTS, + .ztrsm_ilnncopy = ztrsm_ilnncopyTS, + .ztrsm_iltucopy = ztrsm_iltucopyTS, + .ztrsm_iltncopy = ztrsm_iltncopyTS, +#else + .ztrsm_iunucopy = ztrsm_ounucopyTS, + .ztrsm_iunncopy = ztrsm_ounncopyTS, + .ztrsm_iutucopy = ztrsm_outucopyTS, + .ztrsm_iutncopy = ztrsm_outncopyTS, + .ztrsm_ilnucopy = ztrsm_olnucopyTS, + .ztrsm_ilnncopy = ztrsm_olnncopyTS, + .ztrsm_iltucopy = ztrsm_oltucopyTS, + .ztrsm_iltncopy = ztrsm_oltncopyTS, +#endif + .ztrsm_ounucopy = ztrsm_ounucopyTS, + .ztrsm_ounncopy = ztrsm_ounncopyTS, + .ztrsm_outucopy = ztrsm_outucopyTS, + .ztrsm_outncopy = ztrsm_outncopyTS, + .ztrsm_olnucopy = ztrsm_olnucopyTS, + .ztrsm_olnncopy = ztrsm_olnncopyTS, + .ztrsm_oltucopy = ztrsm_oltucopyTS, + .ztrsm_oltncopy = ztrsm_oltncopyTS, + + .ztrmm_kernel_RN = ztrmm_kernel_RNTS, + .ztrmm_kernel_RT = ztrmm_kernel_RTTS, + .ztrmm_kernel_RR = ztrmm_kernel_RRTS, + .ztrmm_kernel_RC = ztrmm_kernel_RCTS, + .ztrmm_kernel_LN = ztrmm_kernel_LNTS, + .ztrmm_kernel_LT = ztrmm_kernel_LTTS, + .ztrmm_kernel_LR = ztrmm_kernel_LRTS, + .ztrmm_kernel_LC = ztrmm_kernel_LCTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - ztrmm_iunucopyTS, ztrmm_iunncopyTS, ztrmm_iutucopyTS, ztrmm_iutncopyTS, - ztrmm_ilnucopyTS, ztrmm_ilnncopyTS, ztrmm_iltucopyTS, ztrmm_iltncopyTS, -#else - ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, - ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, -#endif - zsymm_incopyTS, zsymm_itcopyTS, - ztrmm_incopyTS, ztrmm_itcopyTS, - ztrmm_ounucopyTS, ztrmm_ounncopyTS, ztrmm_outucopyTS, ztrmm_outncopyTS, - ztrmm_olnucopyTS, ztrmm_olnncopyTS, ztrmm_oltucopyTS, ztrmm_oltncopyTS, + .ztrmm_iunucopy = ztrmm_iunucopyTS, + .ztrmm_iunncopy = ztrmm_iunncopyTS, + .ztrmm_iutucopy = ztrmm_iutucopyTS, + .ztrmm_iutncopy = ztrmm_iutncopyTS, + .ztrmm_ilnucopy = ztrmm_ilnucopyTS, + .ztrmm_ilnncopy = ztrmm_ilnncopyTS, + .ztrmm_iltucopy = ztrmm_iltucopyTS, + .ztrmm_iltncopy = ztrmm_iltncopyTS, +#else + .ztrmm_iunucopy = ztrmm_ounucopyTS, + .ztrmm_iunncopy = ztrmm_ounncopyTS, + .ztrmm_iutucopy = ztrmm_outucopyTS, + .ztrmm_iutncopy = ztrmm_outncopyTS, + .ztrmm_ilnucopy = ztrmm_olnucopyTS, + .ztrmm_ilnncopy = ztrmm_olnncopyTS, + .ztrmm_iltucopy = ztrmm_oltucopyTS, + .ztrmm_iltncopy = ztrmm_oltncopyTS, +#endif + .zsymm_incopy = zsymm_incopyTS, + .zsymm_itcopy = zsymm_itcopyTS, + .ztrmm_incopy = ztrmm_incopyTS, + .ztrmm_itcopy = ztrmm_itcopyTS, + .ztrmm_ounucopy = ztrmm_ounucopyTS, + .ztrmm_ounncopy = ztrmm_ounncopyTS, + .ztrmm_outucopy = ztrmm_outucopyTS, + .ztrmm_outncopy = ztrmm_outncopyTS, + .ztrmm_olnucopy = ztrmm_olnucopyTS, + .ztrmm_olnncopy = ztrmm_olnncopyTS, + .ztrmm_oltucopy = ztrmm_oltucopyTS, + .ztrmm_oltncopy = ztrmm_oltncopyTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zsymm_iutcopyTS, zsymm_iltcopyTS, + .zsymm_iutcopy = zsymm_iutcopyTS, + .zsymm_iltcopy = zsymm_iltcopyTS, #else - zsymm_outcopyTS, zsymm_oltcopyTS, + .zsymm_iutcopy = zsymm_outcopyTS, + .zsymm_iltcopy = zsymm_oltcopyTS, #endif - zsymm_outcopyTS, zsymm_oltcopyTS, + .zsymm_outcopy = zsymm_outcopyTS, + .zsymm_oltcopy = zsymm_oltcopyTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - zhemm_iutcopyTS, zhemm_iltcopyTS, + .zhemm_iutcopy = zhemm_iutcopyTS, + .zhemm_iltcopy = zhemm_iltcopyTS, #else - zhemm_outcopyTS, zhemm_oltcopyTS, + .zhemm_iutcopy = zhemm_outcopyTS, + .zhemm_iltcopy = zhemm_oltcopyTS, #endif - zhemm_outcopyTS, zhemm_oltcopyTS, + .zhemm_outcopy = zhemm_outcopyTS, + .zhemm_oltcopy = zhemm_oltcopyTS, - 0, 0, 0, + .zgemm3m_p = 0, + .zgemm3m_q = 0, + .zgemm3m_r = 0, #if (USE_GEMM3M) #ifdef ZGEMM3M_DEFAULT_UNROLL_M - ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N, MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), -#else - DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N, MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), -#endif - - - zgemm3m_kernelTS, - - zgemm3m_incopybTS, zgemm3m_incopyrTS, - zgemm3m_incopyiTS, zgemm3m_itcopybTS, - zgemm3m_itcopyrTS, zgemm3m_itcopyiTS, - zgemm3m_oncopybTS, zgemm3m_oncopyrTS, - zgemm3m_oncopyiTS, zgemm3m_otcopybTS, - zgemm3m_otcopyrTS, zgemm3m_otcopyiTS, - - zsymm3m_iucopybTS, zsymm3m_ilcopybTS, - zsymm3m_iucopyrTS, zsymm3m_ilcopyrTS, - zsymm3m_iucopyiTS, zsymm3m_ilcopyiTS, - zsymm3m_oucopybTS, zsymm3m_olcopybTS, - zsymm3m_oucopyrTS, zsymm3m_olcopyrTS, - zsymm3m_oucopyiTS, zsymm3m_olcopyiTS, - - zhemm3m_iucopybTS, zhemm3m_ilcopybTS, - zhemm3m_iucopyrTS, zhemm3m_ilcopyrTS, - zhemm3m_iucopyiTS, zhemm3m_ilcopyiTS, - - zhemm3m_oucopybTS, zhemm3m_olcopybTS, - zhemm3m_oucopyrTS, zhemm3m_olcopyrTS, - zhemm3m_oucopyiTS, zhemm3m_olcopyiTS, -#else - 0, 0, 0, - - NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, + .zgemm3m_unroll_m = ZGEMM3M_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = ZGEMM3M_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), +#else + .zgemm3m_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = DGEMM_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#endif + + + .zgemm3m_kernel = zgemm3m_kernelTS, + + .zgemm3m_incopyb = zgemm3m_incopybTS, + .zgemm3m_incopyr = zgemm3m_incopyrTS, + .zgemm3m_incopyi = zgemm3m_incopyiTS, + .zgemm3m_itcopyb = zgemm3m_itcopybTS, + .zgemm3m_itcopyr = zgemm3m_itcopyrTS, + .zgemm3m_itcopyi = zgemm3m_itcopyiTS, + .zgemm3m_oncopyb = zgemm3m_oncopybTS, + .zgemm3m_oncopyr = zgemm3m_oncopyrTS, + .zgemm3m_oncopyi = zgemm3m_oncopyiTS, + .zgemm3m_otcopyb = zgemm3m_otcopybTS, + .zgemm3m_otcopyr = zgemm3m_otcopyrTS, + .zgemm3m_otcopyi = zgemm3m_otcopyiTS, + + .zsymm3m_iucopyb = zsymm3m_iucopybTS, + .zsymm3m_ilcopyb = zsymm3m_ilcopybTS, + .zsymm3m_iucopyr = zsymm3m_iucopyrTS, + .zsymm3m_ilcopyr = zsymm3m_ilcopyrTS, + .zsymm3m_iucopyi = zsymm3m_iucopyiTS, + .zsymm3m_ilcopyi = zsymm3m_ilcopyiTS, + .zsymm3m_oucopyb = zsymm3m_oucopybTS, + .zsymm3m_olcopyb = zsymm3m_olcopybTS, + .zsymm3m_oucopyr = zsymm3m_oucopyrTS, + .zsymm3m_olcopyr = zsymm3m_olcopyrTS, + .zsymm3m_oucopyi = zsymm3m_oucopyiTS, + .zsymm3m_olcopyi = zsymm3m_olcopyiTS, + + .zhemm3m_iucopyb = zhemm3m_iucopybTS, + .zhemm3m_ilcopyb = zhemm3m_ilcopybTS, + .zhemm3m_iucopyr = zhemm3m_iucopyrTS, + .zhemm3m_ilcopyr = zhemm3m_ilcopyrTS, + .zhemm3m_iucopyi = zhemm3m_iucopyiTS, + .zhemm3m_ilcopyi = zhemm3m_ilcopyiTS, + + .zhemm3m_oucopyb = zhemm3m_oucopybTS, + .zhemm3m_olcopyb = zhemm3m_olcopybTS, + .zhemm3m_oucopyr = zhemm3m_oucopyrTS, + .zhemm3m_olcopyr = zhemm3m_olcopyrTS, + .zhemm3m_oucopyi = zhemm3m_oucopyiTS, + .zhemm3m_olcopyi = zhemm3m_olcopyiTS, +#else + .zgemm3m_unroll_m = 0, + .zgemm3m_unroll_n = 0, + .zgemm3m_unroll_mn = 0, + + .zgemm3m_kernel = NULL, + + .zgemm3m_incopyb = NULL, + .zgemm3m_incopyr = NULL, + .zgemm3m_incopyi = NULL, + .zgemm3m_itcopyb = NULL, + .zgemm3m_itcopyr = NULL, + .zgemm3m_itcopyi = NULL, + .zgemm3m_oncopyb = NULL, + .zgemm3m_oncopyr = NULL, + .zgemm3m_oncopyi = NULL, + .zgemm3m_otcopyb = NULL, + .zgemm3m_otcopyr = NULL, + .zgemm3m_otcopyi = NULL, + + .zsymm3m_iucopyb = NULL, + .zsymm3m_ilcopyb = NULL, + .zsymm3m_iucopyr = NULL, + .zsymm3m_ilcopyr = NULL, + .zsymm3m_iucopyi = NULL, + .zsymm3m_ilcopyi = NULL, + .zsymm3m_oucopyb = NULL, + .zsymm3m_olcopyb = NULL, + .zsymm3m_oucopyr = NULL, + .zsymm3m_olcopyr = NULL, + .zsymm3m_oucopyi = NULL, + .zsymm3m_olcopyi = NULL, + + .zhemm3m_iucopyb = NULL, + .zhemm3m_ilcopyb = NULL, + .zhemm3m_iucopyr = NULL, + .zhemm3m_ilcopyr = NULL, + .zhemm3m_iucopyi = NULL, + .zhemm3m_ilcopyi = NULL, + + .zhemm3m_oucopyb = NULL, + .zhemm3m_olcopyb = NULL, + .zhemm3m_oucopyr = NULL, + .zhemm3m_olcopyr = NULL, + .zhemm3m_oucopyi = NULL, + .zhemm3m_olcopyi = NULL, #endif #ifndef NO_LAPACK - zneg_tcopyTS, zlaswp_ncopyTS, + .zneg_tcopy = zneg_tcopyTS, + .zlaswp_ncopy = zlaswp_ncopyTS, #else - NULL, NULL, + .zneg_tcopy = NULL, + .zlaswp_ncopy = NULL, #endif #endif #ifdef EXPRECISION - 0, 0, 0, - XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N, MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), - - xamax_kTS, xamin_kTS, ixamax_kTS, ixamin_kTS, - xnrm2_kTS, xasum_kTS, xsum_kTS, xcopy_kTS, - xdotu_kTS, xdotc_kTS, xqrot_kTS, - xaxpy_kTS, xaxpyc_kTS, xscal_kTS, xswap_kTS, - - xgemv_nTS, xgemv_tTS, xgemv_rTS, xgemv_cTS, - xgemv_oTS, xgemv_uTS, xgemv_sTS, xgemv_dTS, - xgeru_kTS, xgerc_kTS, xgerv_kTS, xgerd_kTS, - xsymv_LTS, xsymv_UTS, - xhemv_LTS, xhemv_UTS, xhemv_MTS, xhemv_VTS, - - xgemm_kernel_nTS, xgemm_kernel_lTS, xgemm_kernel_rTS, xgemm_kernel_bTS, - xgemm_betaTS, + .xgemm_p = 0, + .xgemm_q = 0, + .xgemm_r = 0, + .xgemm_unroll_m = XGEMM_DEFAULT_UNROLL_M, + .xgemm_unroll_n = XGEMM_DEFAULT_UNROLL_N, + .xgemm_unroll_mn = MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), + + .xamax_k = xamax_kTS, + .xamin_k = xamin_kTS, + .ixamax_k = ixamax_kTS, + .ixamin_k = ixamin_kTS, + .xnrm2_k = xnrm2_kTS, + .xasum_k = xasum_kTS, + .xsum_k = xsum_kTS, + .xcopy_k = xcopy_kTS, + .xdotu_k = xdotu_kTS, + .xdotc_k = xdotc_kTS, + .xqrot_k = xqrot_kTS, + .xaxpy_k = xaxpy_kTS, + .xaxpyc_k = xaxpyc_kTS, + .xscal_k = xscal_kTS, + .xswap_k = xswap_kTS, + + .xgemv_n = xgemv_nTS, + .xgemv_t = xgemv_tTS, + .xgemv_r = xgemv_rTS, + .xgemv_c = xgemv_cTS, + .xgemv_o = xgemv_oTS, + .xgemv_u = xgemv_uTS, + .xgemv_s = xgemv_sTS, + .xgemv_d = xgemv_dTS, + .xgeru_k = xgeru_kTS, + .xgerc_k = xgerc_kTS, + .xgerv_k = xgerv_kTS, + .xgerd_k = xgerd_kTS, + .xsymv_L = xsymv_LTS, + .xsymv_U = xsymv_UTS, + .xhemv_L = xhemv_LTS, + .xhemv_U = xhemv_UTS, + .xhemv_M = xhemv_MTS, + .xhemv_V = xhemv_VTS, + + .xgemm_kernel_n = xgemm_kernel_nTS, + .xgemm_kernel_l = xgemm_kernel_lTS, + .xgemm_kernel_r = xgemm_kernel_rTS, + .xgemm_kernel_b = xgemm_kernel_bTS, + .xgemm_beta = xgemm_betaTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xgemm_incopyTS, xgemm_itcopyTS, + .xgemm_incopy = xgemm_incopyTS, + .xgemm_itcopy = xgemm_itcopyTS, #else - xgemm_oncopyTS, xgemm_otcopyTS, + .xgemm_incopy = xgemm_oncopyTS, + .xgemm_itcopy = xgemm_otcopyTS, #endif - xgemm_oncopyTS, xgemm_otcopyTS, + .xgemm_oncopy = xgemm_oncopyTS, + .xgemm_otcopy = xgemm_otcopyTS, - xtrsm_kernel_LNTS, xtrsm_kernel_LTTS, xtrsm_kernel_LRTS, xtrsm_kernel_LCTS, - xtrsm_kernel_RNTS, xtrsm_kernel_RTTS, xtrsm_kernel_RRTS, xtrsm_kernel_RCTS, + .xtrsm_kernel_LN = xtrsm_kernel_LNTS, + .xtrsm_kernel_LT = xtrsm_kernel_LTTS, + .xtrsm_kernel_LR = xtrsm_kernel_LRTS, + .xtrsm_kernel_LC = xtrsm_kernel_LCTS, + .xtrsm_kernel_RN = xtrsm_kernel_RNTS, + .xtrsm_kernel_RT = xtrsm_kernel_RTTS, + .xtrsm_kernel_RR = xtrsm_kernel_RRTS, + .xtrsm_kernel_RC = xtrsm_kernel_RCTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xtrsm_iunucopyTS, xtrsm_iunncopyTS, xtrsm_iutucopyTS, xtrsm_iutncopyTS, - xtrsm_ilnucopyTS, xtrsm_ilnncopyTS, xtrsm_iltucopyTS, xtrsm_iltncopyTS, -#else - xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS, - xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS, -#endif - xtrsm_ounucopyTS, xtrsm_ounncopyTS, xtrsm_outucopyTS, xtrsm_outncopyTS, - xtrsm_olnucopyTS, xtrsm_olnncopyTS, xtrsm_oltucopyTS, xtrsm_oltncopyTS, - - xtrmm_kernel_RNTS, xtrmm_kernel_RTTS, xtrmm_kernel_RRTS, xtrmm_kernel_RCTS, - xtrmm_kernel_LNTS, xtrmm_kernel_LTTS, xtrmm_kernel_LRTS, xtrmm_kernel_LCTS, + .xtrsm_iunucopy = xtrsm_iunucopyTS, + .xtrsm_iunncopy = xtrsm_iunncopyTS, + .xtrsm_iutucopy = xtrsm_iutucopyTS, + .xtrsm_iutncopy = xtrsm_iutncopyTS, + .xtrsm_ilnucopy = xtrsm_ilnucopyTS, + .xtrsm_ilnncopy = xtrsm_ilnncopyTS, + .xtrsm_iltucopy = xtrsm_iltucopyTS, + .xtrsm_iltncopy = xtrsm_iltncopyTS, +#else + .xtrsm_iunucopy = xtrsm_ounucopyTS, + .xtrsm_iunncopy = xtrsm_ounncopyTS, + .xtrsm_iutucopy = xtrsm_outucopyTS, + .xtrsm_iutncopy = xtrsm_outncopyTS, + .xtrsm_ilnucopy = xtrsm_olnucopyTS, + .xtrsm_ilnncopy = xtrsm_olnncopyTS, + .xtrsm_iltucopy = xtrsm_oltucopyTS, + .xtrsm_iltncopy = xtrsm_oltncopyTS, +#endif + .xtrsm_ounucopy = xtrsm_ounucopyTS, + .xtrsm_ounncopy = xtrsm_ounncopyTS, + .xtrsm_outucopy = xtrsm_outucopyTS, + .xtrsm_outncopy = xtrsm_outncopyTS, + .xtrsm_olnucopy = xtrsm_olnucopyTS, + .xtrsm_olnncopy = xtrsm_olnncopyTS, + .xtrsm_oltucopy = xtrsm_oltucopyTS, + .xtrsm_oltncopy = xtrsm_oltncopyTS, + + .xtrmm_kernel_RN = xtrmm_kernel_RNTS, + .xtrmm_kernel_RT = xtrmm_kernel_RTTS, + .xtrmm_kernel_RR = xtrmm_kernel_RRTS, + .xtrmm_kernel_RC = xtrmm_kernel_RCTS, + .xtrmm_kernel_LN = xtrmm_kernel_LNTS, + .xtrmm_kernel_LT = xtrmm_kernel_LTTS, + .xtrmm_kernel_LR = xtrmm_kernel_LRTS, + .xtrmm_kernel_LC = xtrmm_kernel_LCTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xtrmm_iunucopyTS, xtrmm_iunncopyTS, xtrmm_iutucopyTS, xtrmm_iutncopyTS, - xtrmm_ilnucopyTS, xtrmm_ilnncopyTS, xtrmm_iltucopyTS, xtrmm_iltncopyTS, -#else - xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS, - xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS, -#endif - xtrmm_ounucopyTS, xtrmm_ounncopyTS, xtrmm_outucopyTS, xtrmm_outncopyTS, - xtrmm_olnucopyTS, xtrmm_olnncopyTS, xtrmm_oltucopyTS, xtrmm_oltncopyTS, + .xtrmm_iunucopy = xtrmm_iunucopyTS, + .xtrmm_iunncopy = xtrmm_iunncopyTS, + .xtrmm_iutucopy = xtrmm_iutucopyTS, + .xtrmm_iutncopy = xtrmm_iutncopyTS, + .xtrmm_ilnucopy = xtrmm_ilnucopyTS, + .xtrmm_ilnncopy = xtrmm_ilnncopyTS, + .xtrmm_iltucopy = xtrmm_iltucopyTS, + .xtrmm_iltncopy = xtrmm_iltncopyTS, +#else + .xtrmm_iunucopy = xtrmm_ounucopyTS, + .xtrmm_iunncopy = xtrmm_ounncopyTS, + .xtrmm_iutucopy = xtrmm_outucopyTS, + .xtrmm_iutncopy = xtrmm_outncopyTS, + .xtrmm_ilnucopy = xtrmm_olnucopyTS, + .xtrmm_ilnncopy = xtrmm_olnncopyTS, + .xtrmm_iltucopy = xtrmm_oltucopyTS, + .xtrmm_iltncopy = xtrmm_oltncopyTS, +#endif + .xtrmm_ounucopy = xtrmm_ounucopyTS, + .xtrmm_ounncopy = xtrmm_ounncopyTS, + .xtrmm_outucopy = xtrmm_outucopyTS, + .xtrmm_outncopy = xtrmm_outncopyTS, + .xtrmm_olnucopy = xtrmm_olnucopyTS, + .xtrmm_olnncopy = xtrmm_olnncopyTS, + .xtrmm_oltucopy = xtrmm_oltucopyTS, + .xtrmm_oltncopy = xtrmm_oltncopyTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xsymm_iutcopyTS, xsymm_iltcopyTS, + .xsymm_iutcopy = xsymm_iutcopyTS, + .xsymm_iltcopy = xsymm_iltcopyTS, #else - xsymm_outcopyTS, xsymm_oltcopyTS, + .xsymm_iutcopy = xsymm_outcopyTS, + .xsymm_iltcopy = xsymm_oltcopyTS, #endif - xsymm_outcopyTS, xsymm_oltcopyTS, + .xsymm_outcopy = xsymm_outcopyTS, + .xsymm_oltcopy = xsymm_oltcopyTS, #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N - xhemm_iutcopyTS, xhemm_iltcopyTS, + .xhemm_iutcopy = xhemm_iutcopyTS, + .xhemm_iltcopy = xhemm_iltcopyTS, #else - xhemm_outcopyTS, xhemm_oltcopyTS, + .xhemm_iutcopy = xhemm_outcopyTS, + .xhemm_iltcopy = xhemm_oltcopyTS, #endif - xhemm_outcopyTS, xhemm_oltcopyTS, + .xhemm_outcopy = xhemm_outcopyTS, + .xhemm_oltcopy = xhemm_oltcopyTS, - 0, 0, 0, + .xgemm3m_p = 0, + .xgemm3m_q = 0, + .xgemm3m_r = 0, #if (USE_GEMM3M) - QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N, MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), - - xgemm3m_kernelTS, - - xgemm3m_incopybTS, xgemm3m_incopyrTS, - xgemm3m_incopyiTS, xgemm3m_itcopybTS, - xgemm3m_itcopyrTS, xgemm3m_itcopyiTS, - xgemm3m_oncopybTS, xgemm3m_oncopyrTS, - xgemm3m_oncopyiTS, xgemm3m_otcopybTS, - xgemm3m_otcopyrTS, xgemm3m_otcopyiTS, - - xsymm3m_iucopybTS, xsymm3m_ilcopybTS, - xsymm3m_iucopyrTS, xsymm3m_ilcopyrTS, - xsymm3m_iucopyiTS, xsymm3m_ilcopyiTS, - xsymm3m_oucopybTS, xsymm3m_olcopybTS, - xsymm3m_oucopyrTS, xsymm3m_olcopyrTS, - xsymm3m_oucopyiTS, xsymm3m_olcopyiTS, - - xhemm3m_iucopybTS, xhemm3m_ilcopybTS, - xhemm3m_iucopyrTS, xhemm3m_ilcopyrTS, - xhemm3m_iucopyiTS, xhemm3m_ilcopyiTS, - - xhemm3m_oucopybTS, xhemm3m_olcopybTS, - xhemm3m_oucopyrTS, xhemm3m_olcopyrTS, - xhemm3m_oucopyiTS, xhemm3m_olcopyiTS, -#else - 0, 0, 0, - - NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, - - NULL, NULL, - NULL, NULL, - NULL, NULL, + .xgemm3m_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .xgemm3m_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .xgemm3m_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), + + .xgemm3m_kernel = xgemm3m_kernelTS, + + .xgemm3m_incopyb = xgemm3m_incopybTS, + .xgemm3m_incopyr = xgemm3m_incopyrTS, + .xgemm3m_incopyi = xgemm3m_incopyiTS, + .xgemm3m_itcopyb = xgemm3m_itcopybTS, + .xgemm3m_itcopyr = xgemm3m_itcopyrTS, + .xgemm3m_itcopyi = xgemm3m_itcopyiTS, + .xgemm3m_oncopyb = xgemm3m_oncopybTS, + .xgemm3m_oncopyr = xgemm3m_oncopyrTS, + .xgemm3m_oncopyi = xgemm3m_oncopyiTS, + .xgemm3m_otcopyb = xgemm3m_otcopybTS, + .xgemm3m_otcopyr = xgemm3m_otcopyrTS, + .xgemm3m_otcopyi = xgemm3m_otcopyiTS, + + .xsymm3m_iucopyb = xsymm3m_iucopybTS, + .xsymm3m_ilcopyb = xsymm3m_ilcopybTS, + .xsymm3m_iucopyr = xsymm3m_iucopyrTS, + .xsymm3m_ilcopyr = xsymm3m_ilcopyrTS, + .xsymm3m_iucopyi = xsymm3m_iucopyiTS, + .xsymm3m_ilcopyi = xsymm3m_ilcopyiTS, + .xsymm3m_oucopyb = xsymm3m_oucopybTS, + .xsymm3m_olcopyb = xsymm3m_olcopybTS, + .xsymm3m_oucopyr = xsymm3m_oucopyrTS, + .xsymm3m_olcopyr = xsymm3m_olcopyrTS, + .xsymm3m_oucopyi = xsymm3m_oucopyiTS, + .xsymm3m_olcopyi = xsymm3m_olcopyiTS, + + .xhemm3m_iucopyb = xhemm3m_iucopybTS, + .xhemm3m_ilcopyb = xhemm3m_ilcopybTS, + .xhemm3m_iucopyr = xhemm3m_iucopyrTS, + .xhemm3m_ilcopyr = xhemm3m_ilcopyrTS, + .xhemm3m_iucopyi = xhemm3m_iucopyiTS, + .xhemm3m_ilcopyi = xhemm3m_ilcopyiTS, + + .xhemm3m_oucopyb = xhemm3m_oucopybTS, + .xhemm3m_olcopyb = xhemm3m_olcopybTS, + .xhemm3m_oucopyr = xhemm3m_oucopyrTS, + .xhemm3m_olcopyr = xhemm3m_olcopyrTS, + .xhemm3m_oucopyi = xhemm3m_oucopyiTS, + .xhemm3m_olcopyi = xhemm3m_olcopyiTS, +#else + .xgemm3m_unroll_m = 0, + .xgemm3m_unroll_n = 0, + .xgemm3m_unroll_mn = 0, + + .xgemm3m_kernel = NULL, + + .xgemm3m_incopyb = NULL, + .xgemm3m_incopyr = NULL, + .xgemm3m_incopyi = NULL, + .xgemm3m_itcopyb = NULL, + .xgemm3m_itcopyr = NULL, + .xgemm3m_itcopyi = NULL, + .xgemm3m_oncopyb = NULL, + .xgemm3m_oncopyr = NULL, + .xgemm3m_oncopyi = NULL, + .xgemm3m_otcopyb = NULL, + .xgemm3m_otcopyr = NULL, + .xgemm3m_otcopyi = NULL, + + .xsymm3m_iucopyb = NULL, + .xsymm3m_ilcopyb = NULL, + .xsymm3m_iucopyr = NULL, + .xsymm3m_ilcopyr = NULL, + .xsymm3m_iucopyi = NULL, + .xsymm3m_ilcopyi = NULL, + .xsymm3m_oucopyb = NULL, + .xsymm3m_olcopyb = NULL, + .xsymm3m_oucopyr = NULL, + .xsymm3m_olcopyr = NULL, + .xsymm3m_oucopyi = NULL, + .xsymm3m_olcopyi = NULL, + + .xhemm3m_iucopyb = NULL, + .xhemm3m_ilcopyb = NULL, + .xhemm3m_iucopyr = NULL, + .xhemm3m_ilcopyr = NULL, + .xhemm3m_iucopyi = NULL, + .xhemm3m_ilcopyi = NULL, + + .xhemm3m_oucopyb = NULL, + .xhemm3m_olcopyb = NULL, + .xhemm3m_oucopyr = NULL, + .xhemm3m_olcopyr = NULL, + .xhemm3m_oucopyi = NULL, + .xhemm3m_olcopyi = NULL, #endif #ifndef NO_LAPACK - xneg_tcopyTS, xlaswp_ncopyTS, + .xneg_tcopy = xneg_tcopyTS, + .xlaswp_ncopy = xlaswp_ncopyTS, #else - NULL, NULL, + .xneg_tcopy = NULL, + .xlaswp_ncopy = NULL, #endif #endif - init_parameter, + .init = init_parameter, - SNUMOPT, DNUMOPT, QNUMOPT, + .snum_opt = SNUMOPT, + .dnum_opt = DNUMOPT, + .qnum_opt = QNUMOPT, #if BUILD_SINGLE == 1 - saxpby_kTS, + .saxpby_k = saxpby_kTS, #endif #if BUILD_DOUBLE == 1 - daxpby_kTS, + .daxpby_k = daxpby_kTS, #endif #if BUILD_COMPLEX == 1 - caxpby_kTS, + .caxpby_k = caxpby_kTS, #endif #if BUILD_COMPLEX16== 1 - zaxpby_kTS, + .zaxpby_k = zaxpby_kTS, #endif #if BUILD_SINGLE == 1 - somatcopy_k_cnTS, somatcopy_k_ctTS, somatcopy_k_rnTS, somatcopy_k_rtTS, + .somatcopy_k_cn = somatcopy_k_cnTS, + .somatcopy_k_ct = somatcopy_k_ctTS, + .somatcopy_k_rn = somatcopy_k_rnTS, + .somatcopy_k_rt = somatcopy_k_rtTS, #endif #if BUILD_DOUBLE== 1 - domatcopy_k_cnTS, domatcopy_k_ctTS, domatcopy_k_rnTS, domatcopy_k_rtTS, + .domatcopy_k_cn = domatcopy_k_cnTS, + .domatcopy_k_ct = domatcopy_k_ctTS, + .domatcopy_k_rn = domatcopy_k_rnTS, + .domatcopy_k_rt = domatcopy_k_rtTS, #endif #if BUILD_COMPLEX == 1 - comatcopy_k_cnTS, comatcopy_k_ctTS, comatcopy_k_rnTS, comatcopy_k_rtTS, - comatcopy_k_cncTS, comatcopy_k_ctcTS, comatcopy_k_rncTS, comatcopy_k_rtcTS, + .comatcopy_k_cn = comatcopy_k_cnTS, + .comatcopy_k_ct = comatcopy_k_ctTS, + .comatcopy_k_rn = comatcopy_k_rnTS, + .comatcopy_k_rt = comatcopy_k_rtTS, + .comatcopy_k_cnc = comatcopy_k_cncTS, + .comatcopy_k_ctc = comatcopy_k_ctcTS, + .comatcopy_k_rnc = comatcopy_k_rncTS, + .comatcopy_k_rtc = comatcopy_k_rtcTS, #endif #if BUILD_COMPLEX16 == 1 - zomatcopy_k_cnTS, zomatcopy_k_ctTS, zomatcopy_k_rnTS, zomatcopy_k_rtTS, - zomatcopy_k_cncTS, zomatcopy_k_ctcTS, zomatcopy_k_rncTS, zomatcopy_k_rtcTS, + .zomatcopy_k_cn = zomatcopy_k_cnTS, + .zomatcopy_k_ct = zomatcopy_k_ctTS, + .zomatcopy_k_rn = zomatcopy_k_rnTS, + .zomatcopy_k_rt = zomatcopy_k_rtTS, + .zomatcopy_k_cnc = zomatcopy_k_cncTS, + .zomatcopy_k_ctc = zomatcopy_k_ctcTS, + .zomatcopy_k_rnc = zomatcopy_k_rncTS, + .zomatcopy_k_rtc = zomatcopy_k_rtcTS, #endif #if BUILD_SINGLE == 1 - simatcopy_k_cnTS, simatcopy_k_ctTS, simatcopy_k_rnTS, simatcopy_k_rtTS, + .simatcopy_k_cn = simatcopy_k_cnTS, + .simatcopy_k_ct = simatcopy_k_ctTS, + .simatcopy_k_rn = simatcopy_k_rnTS, + .simatcopy_k_rt = simatcopy_k_rtTS, #endif #if BUILD_DOUBLE== 1 - dimatcopy_k_cnTS, dimatcopy_k_ctTS, dimatcopy_k_rnTS, dimatcopy_k_rtTS, + .dimatcopy_k_cn = dimatcopy_k_cnTS, + .dimatcopy_k_ct = dimatcopy_k_ctTS, + .dimatcopy_k_rn = dimatcopy_k_rnTS, + .dimatcopy_k_rt = dimatcopy_k_rtTS, #endif #if BUILD_COMPLEX== 1 - cimatcopy_k_cnTS, cimatcopy_k_ctTS, cimatcopy_k_rnTS, cimatcopy_k_rtTS, - cimatcopy_k_cncTS, cimatcopy_k_ctcTS, cimatcopy_k_rncTS, cimatcopy_k_rtcTS, + .cimatcopy_k_cn = cimatcopy_k_cnTS, + .cimatcopy_k_ct = cimatcopy_k_ctTS, + .cimatcopy_k_rn = cimatcopy_k_rnTS, + .cimatcopy_k_rt = cimatcopy_k_rtTS, + .cimatcopy_k_cnc = cimatcopy_k_cncTS, + .cimatcopy_k_ctc = cimatcopy_k_ctcTS, + .cimatcopy_k_rnc = cimatcopy_k_rncTS, + .cimatcopy_k_rtc = cimatcopy_k_rtcTS, #endif #if BUILD_COMPLEX16==1 - zimatcopy_k_cnTS, zimatcopy_k_ctTS, zimatcopy_k_rnTS, zimatcopy_k_rtTS, - zimatcopy_k_cncTS, zimatcopy_k_ctcTS, zimatcopy_k_rncTS, zimatcopy_k_rtcTS, + .zimatcopy_k_cn = zimatcopy_k_cnTS, + .zimatcopy_k_ct = zimatcopy_k_ctTS, + .zimatcopy_k_rn = zimatcopy_k_rnTS, + .zimatcopy_k_rt = zimatcopy_k_rtTS, + .zimatcopy_k_cnc = zimatcopy_k_cncTS, + .zimatcopy_k_ctc = zimatcopy_k_ctcTS, + .zimatcopy_k_rnc = zimatcopy_k_rncTS, + .zimatcopy_k_rtc = zimatcopy_k_rtcTS, #endif #if BUILD_SINGLE == 1 - sgeadd_kTS, + .sgeadd_k = sgeadd_kTS, #endif #if BUILD_DOUBLE==1 - dgeadd_kTS, + .dgeadd_k = dgeadd_kTS, #endif #if BUILD_COMPLEX==1 - cgeadd_kTS, + .cgeadd_k = cgeadd_kTS, #endif #if BUILD_COMPLEX16==1 - zgeadd_kTS, + .zgeadd_k = zgeadd_kTS, #endif }; From 2f8f611293f183a93e664041f1367c0562b669cf Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 23:18:41 +0000 Subject: [PATCH 6/7] DYNAMIC_ARCH: split the kernels out of gotoblas_t into per-group tables With DYNAMIC_ARCH, gotoblas_t holds every kernel of a core, and the cpu detection code in driver/others/dynamic*.c takes the address of every core's gotoblas_t. One reference to a core's table thus keeps all of that core's kernels, and a static link keeps every kernel of every core, of which --gc-sections can remove almost nothing. Move the 904 kernel pointers into 233 small tables, one per kernel group (the kernels whose names share a leading token: dgemm_kernel, dgemm_beta, dgemm_incopy, ... are "dgemm"), each instantiated per core in setparam-ref.c and reached through OPENBLAS_DISPATCH(group), from an array of them that driver/others/dispatch.c defines. gotoblas_t keeps only the tuning parameters and init(), so cpu detection no longer references any kernel, and a program links only the groups it calls: text of a static executable before after dgemm only 19.5 MB 0.7 MB dgesv + daxpy + dgemv + dnrm2 19.5 MB 1.6 MB every cblas_d* routine 19.8 MB 3.5 MB (x86_64, 14 cores, linked with --gc-sections; dispatch-split/measure-size.sh.) This commit is the output of dispatch-split/split_dispatch_groups.py and nothing else; see there for the exact rules. "gotoblas -> dgemm_kernel" becomes "OPENBLAS_DISPATCH(dgemm) -> dgemm_kernel", that is openblas_dgemm_dispatch[gotoblas->core]->dgemm_kernel. That costs nothing on real work (single-threaded dgemm, n=2048: 57.1 GFLOP/s before, 57.2 after) but shows on the smallest calls (ddot, n=8: 5.05 ns per call before, 5.6-5.8 ns after). For all 14 x86_64 cores, every kernel pointer (resolved to its symbol) and every tuning parameter (after init()) is the same as before, 11830 entries in all (dispatch-split/compare-tables.py). Generated-by: dispatch-split/split_dispatch_groups.py Co-Authored-By: Claude Opus 5.5 --- common_b.h | 18 +- common_c.h | 472 ++++----- common_d.h | 274 ++--- common_param.h | 1730 +++++++++++++++++++++++++------ common_q.h | 212 ++-- common_s.h | 318 +++--- common_sb.h | 46 +- common_sh.h | 20 +- common_x.h | 314 +++--- common_z.h | 472 ++++----- driver/others/dispatch.c | 936 +++++++++++++++++ kernel/setparam-ref.c | 2065 ++++++++++++++++++++++++++++---------- 12 files changed, 4995 insertions(+), 1882 deletions(-) diff --git a/common_b.h b/common_b.h index 1921c3a69d..780dcbc240 100644 --- a/common_b.h +++ b/common_b.h @@ -50,17 +50,17 @@ #define BGEMM_KERNEL bgemm_kernel #else -#define BGEMV_N_K gotoblas->bgemv_n -#define BGEMV_T_K gotoblas->bgemv_t +#define BGEMV_N_K OPENBLAS_DISPATCH(bgemv)->bgemv_n +#define BGEMV_T_K OPENBLAS_DISPATCH(bgemv)->bgemv_t -#define BSCAL_K gotoblas->bscal_k +#define BSCAL_K OPENBLAS_DISPATCH(bscal)->bscal_k -#define BGEMM_ONCOPY gotoblas->bgemm_oncopy -#define BGEMM_OTCOPY gotoblas->bgemm_otcopy -#define BGEMM_INCOPY gotoblas->bgemm_incopy -#define BGEMM_ITCOPY gotoblas->bgemm_itcopy -#define BGEMM_BETA gotoblas->bgemm_beta -#define BGEMM_KERNEL gotoblas->bgemm_kernel +#define BGEMM_ONCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_oncopy +#define BGEMM_OTCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_otcopy +#define BGEMM_INCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_incopy +#define BGEMM_ITCOPY OPENBLAS_DISPATCH(bgemm)->bgemm_itcopy +#define BGEMM_BETA OPENBLAS_DISPATCH(bgemm)->bgemm_beta +#define BGEMM_KERNEL OPENBLAS_DISPATCH(bgemm)->bgemm_kernel #endif diff --git a/common_c.h b/common_c.h index d0a4e595fb..40f30afea7 100644 --- a/common_c.h +++ b/common_c.h @@ -290,46 +290,46 @@ #else -#define CAMAX_K gotoblas -> camax_k -#define CAMIN_K gotoblas -> camin_k +#define CAMAX_K OPENBLAS_DISPATCH(camax) -> camax_k +#define CAMIN_K OPENBLAS_DISPATCH(camin) -> camin_k #define CMAX_K gotoblas -> cmax_k #define CMIN_K gotoblas -> cmin_k -#define ICAMAX_K gotoblas -> icamax_k -#define ICAMIN_K gotoblas -> icamin_k +#define ICAMAX_K OPENBLAS_DISPATCH(icamax) -> icamax_k +#define ICAMIN_K OPENBLAS_DISPATCH(icamin) -> icamin_k #define ICMAX_K gotoblas -> icmax_k #define ICMIN_K gotoblas -> icmin_k -#define CASUM_K gotoblas -> casum_k -#define CAXPYU_K gotoblas -> caxpy_k -#define CAXPYC_K gotoblas -> caxpyc_k -#define CCOPY_K gotoblas -> ccopy_k -#define CDOTU_K gotoblas -> cdotu_k -#define CDOTC_K gotoblas -> cdotc_k -#define CNRM2_K gotoblas -> cnrm2_k -#define CSCAL_K gotoblas -> cscal_k -#define CSUM_K gotoblas -> csum_k -#define CSWAP_K gotoblas -> cswap_k -#define CROT_K gotoblas -> csrot_k - -#define CGEMV_N gotoblas -> cgemv_n -#define CGEMV_T gotoblas -> cgemv_t -#define CGEMV_R gotoblas -> cgemv_r -#define CGEMV_C gotoblas -> cgemv_c -#define CGEMV_O gotoblas -> cgemv_o -#define CGEMV_U gotoblas -> cgemv_u -#define CGEMV_S gotoblas -> cgemv_s -#define CGEMV_D gotoblas -> cgemv_d - -#define CGERU_K gotoblas -> cgeru_k -#define CGERC_K gotoblas -> cgerc_k -#define CGERV_K gotoblas -> cgerv_k -#define CGERD_K gotoblas -> cgerd_k - -#define CSYMV_U gotoblas -> csymv_U -#define CSYMV_L gotoblas -> csymv_L -#define CHEMV_U gotoblas -> chemv_U -#define CHEMV_L gotoblas -> chemv_L -#define CHEMV_V gotoblas -> chemv_V -#define CHEMV_M gotoblas -> chemv_M +#define CASUM_K OPENBLAS_DISPATCH(casum) -> casum_k +#define CAXPYU_K OPENBLAS_DISPATCH(caxpy) -> caxpy_k +#define CAXPYC_K OPENBLAS_DISPATCH(caxpyc) -> caxpyc_k +#define CCOPY_K OPENBLAS_DISPATCH(ccopy) -> ccopy_k +#define CDOTU_K OPENBLAS_DISPATCH(cdotu) -> cdotu_k +#define CDOTC_K OPENBLAS_DISPATCH(cdotc) -> cdotc_k +#define CNRM2_K OPENBLAS_DISPATCH(cnrm2) -> cnrm2_k +#define CSCAL_K OPENBLAS_DISPATCH(cscal) -> cscal_k +#define CSUM_K OPENBLAS_DISPATCH(csum) -> csum_k +#define CSWAP_K OPENBLAS_DISPATCH(cswap) -> cswap_k +#define CROT_K OPENBLAS_DISPATCH(csrot) -> csrot_k + +#define CGEMV_N OPENBLAS_DISPATCH(cgemv) -> cgemv_n +#define CGEMV_T OPENBLAS_DISPATCH(cgemv) -> cgemv_t +#define CGEMV_R OPENBLAS_DISPATCH(cgemv) -> cgemv_r +#define CGEMV_C OPENBLAS_DISPATCH(cgemv) -> cgemv_c +#define CGEMV_O OPENBLAS_DISPATCH(cgemv) -> cgemv_o +#define CGEMV_U OPENBLAS_DISPATCH(cgemv) -> cgemv_u +#define CGEMV_S OPENBLAS_DISPATCH(cgemv) -> cgemv_s +#define CGEMV_D OPENBLAS_DISPATCH(cgemv) -> cgemv_d + +#define CGERU_K OPENBLAS_DISPATCH(cgeru) -> cgeru_k +#define CGERC_K OPENBLAS_DISPATCH(cgerc) -> cgerc_k +#define CGERV_K OPENBLAS_DISPATCH(cgerv) -> cgerv_k +#define CGERD_K OPENBLAS_DISPATCH(cgerd) -> cgerd_k + +#define CSYMV_U OPENBLAS_DISPATCH(csymv) -> csymv_U +#define CSYMV_L OPENBLAS_DISPATCH(csymv) -> csymv_L +#define CHEMV_U OPENBLAS_DISPATCH(chemv) -> chemv_U +#define CHEMV_L OPENBLAS_DISPATCH(chemv) -> chemv_L +#define CHEMV_V OPENBLAS_DISPATCH(chemv) -> chemv_V +#define CHEMV_M OPENBLAS_DISPATCH(chemv) -> chemv_M #define CSYMV_THREAD_U csymv_thread_U #define CSYMV_THREAD_L csymv_thread_L @@ -338,209 +338,209 @@ #define CHEMV_THREAD_V chemv_thread_V #define CHEMV_THREAD_M chemv_thread_M -#define CGEMM_ONCOPY gotoblas -> cgemm_oncopy -#define CGEMM_OTCOPY gotoblas -> cgemm_otcopy -#define CGEMM_INCOPY gotoblas -> cgemm_incopy -#define CGEMM_ITCOPY gotoblas -> cgemm_itcopy - -#define CTRMM_OUNUCOPY gotoblas -> ctrmm_ounucopy -#define CTRMM_OUTUCOPY gotoblas -> ctrmm_outucopy -#define CTRMM_OLNUCOPY gotoblas -> ctrmm_olnucopy -#define CTRMM_OLTUCOPY gotoblas -> ctrmm_oltucopy -#define CTRSM_OUNUCOPY gotoblas -> ctrsm_ounucopy -#define CTRSM_OUTUCOPY gotoblas -> ctrsm_outucopy -#define CTRSM_OLNUCOPY gotoblas -> ctrsm_olnucopy -#define CTRSM_OLTUCOPY gotoblas -> ctrsm_oltucopy - -#define CTRMM_IUNUCOPY gotoblas -> ctrmm_iunucopy -#define CTRMM_IUTUCOPY gotoblas -> ctrmm_iutucopy -#define CTRMM_ILNUCOPY gotoblas -> ctrmm_ilnucopy -#define CTRMM_ILTUCOPY gotoblas -> ctrmm_iltucopy -#define CTRSM_IUNUCOPY gotoblas -> ctrsm_iunucopy -#define CTRSM_IUTUCOPY gotoblas -> ctrsm_iutucopy -#define CTRSM_ILNUCOPY gotoblas -> ctrsm_ilnucopy -#define CTRSM_ILTUCOPY gotoblas -> ctrsm_iltucopy - -#define CTRMM_OUNNCOPY gotoblas -> ctrmm_ounncopy -#define CTRMM_OUTNCOPY gotoblas -> ctrmm_outncopy -#define CTRMM_OLNNCOPY gotoblas -> ctrmm_olnncopy -#define CTRMM_OLTNCOPY gotoblas -> ctrmm_oltncopy -#define CTRSM_OUNNCOPY gotoblas -> ctrsm_ounncopy -#define CTRSM_OUTNCOPY gotoblas -> ctrsm_outncopy -#define CTRSM_OLNNCOPY gotoblas -> ctrsm_olnncopy -#define CTRSM_OLTNCOPY gotoblas -> ctrsm_oltncopy - -#define CTRMM_IUNNCOPY gotoblas -> ctrmm_iunncopy -#define CTRMM_IUTNCOPY gotoblas -> ctrmm_iutncopy -#define CTRMM_ILNNCOPY gotoblas -> ctrmm_ilnncopy -#define CTRMM_ILTNCOPY gotoblas -> ctrmm_iltncopy -#define CSYMM_INCOPY gotoblas -> csymm_incopy -#define CSYMM_ITCOPY gotoblas -> csymm_itcopy -#define CTRMM_INCOPY gotoblas -> ctrmm_incopy -#define CTRMM_ITCOPY gotoblas -> ctrmm_itcopy - -#define CTRSM_IUNNCOPY gotoblas -> ctrsm_iunncopy -#define CTRSM_IUTNCOPY gotoblas -> ctrsm_iutncopy -#define CTRSM_ILNNCOPY gotoblas -> ctrsm_ilnncopy -#define CTRSM_ILTNCOPY gotoblas -> ctrsm_iltncopy - -#define SME_CGEMM_KERNEL gotoblas -> sme_cgemm_kernel -#define CGEMM_BETA gotoblas -> cgemm_beta -#define CGEMM_KERNEL_N gotoblas -> cgemm_kernel_n -#define CGEMM_KERNEL_L gotoblas -> cgemm_kernel_l -#define CGEMM_KERNEL_R gotoblas -> cgemm_kernel_r -#define CGEMM_KERNEL_B gotoblas -> cgemm_kernel_b - -#define CSYMM_KERNEL_N gotoblas -> csymm_kernel_n -#define CSYMM_KERNEL_L gotoblas -> csymm_kernel_l -#define CSYMM_KERNEL_R gotoblas -> csymm_kernel_r -#define CSYMM_KERNEL_B gotoblas -> csymm_kernel_b -#define CTRMM_GEMM_KERNEL_N gotoblas -> ctrmm_gemm_kernel_n -#define CTRMM_GEMM_KERNEL_L gotoblas -> ctrmm_gemm_kernel_l -#define CTRMM_GEMM_KERNEL_R gotoblas -> ctrmm_gemm_kernel_r -#define CTRMM_GEMM_KERNEL_B gotoblas -> ctrmm_gemm_kernel_b - -#define CTRMM_KERNEL_LN gotoblas -> ctrmm_kernel_LN -#define CTRMM_KERNEL_LT gotoblas -> ctrmm_kernel_LT -#define CTRMM_KERNEL_LR gotoblas -> ctrmm_kernel_LR -#define CTRMM_KERNEL_LC gotoblas -> ctrmm_kernel_LC -#define CTRMM_KERNEL_RN gotoblas -> ctrmm_kernel_RN -#define CTRMM_KERNEL_RT gotoblas -> ctrmm_kernel_RT -#define CTRMM_KERNEL_RR gotoblas -> ctrmm_kernel_RR -#define CTRMM_KERNEL_RC gotoblas -> ctrmm_kernel_RC - -#define CTRSM_KERNEL_LN gotoblas -> ctrsm_kernel_LN -#define CTRSM_KERNEL_LT gotoblas -> ctrsm_kernel_LT -#define CTRSM_KERNEL_LR gotoblas -> ctrsm_kernel_LR -#define CTRSM_KERNEL_LC gotoblas -> ctrsm_kernel_LC -#define CTRSM_KERNEL_RN gotoblas -> ctrsm_kernel_RN -#define CTRSM_KERNEL_RT gotoblas -> ctrsm_kernel_RT -#define CTRSM_KERNEL_RR gotoblas -> ctrsm_kernel_RR -#define CTRSM_KERNEL_RC gotoblas -> ctrsm_kernel_RC - -#define CSYMM_IUTCOPY gotoblas -> csymm_iutcopy -#define CSYMM_ILTCOPY gotoblas -> csymm_iltcopy -#define CSYMM_OUTCOPY gotoblas -> csymm_outcopy -#define CSYMM_OLTCOPY gotoblas -> csymm_oltcopy - -#define CHEMM_OUTCOPY gotoblas -> chemm_outcopy -#define CHEMM_OLTCOPY gotoblas -> chemm_oltcopy -#define CHEMM_IUTCOPY gotoblas -> chemm_iutcopy -#define CHEMM_ILTCOPY gotoblas -> chemm_iltcopy - -#define CGEMM3M_ONCOPYB gotoblas -> cgemm3m_oncopyb -#define CGEMM3M_ONCOPYR gotoblas -> cgemm3m_oncopyr -#define CGEMM3M_ONCOPYI gotoblas -> cgemm3m_oncopyi -#define CGEMM3M_OTCOPYB gotoblas -> cgemm3m_otcopyb -#define CGEMM3M_OTCOPYR gotoblas -> cgemm3m_otcopyr -#define CGEMM3M_OTCOPYI gotoblas -> cgemm3m_otcopyi - -#define CGEMM3M_INCOPYB gotoblas -> cgemm3m_incopyb -#define CGEMM3M_INCOPYR gotoblas -> cgemm3m_incopyr -#define CGEMM3M_INCOPYI gotoblas -> cgemm3m_incopyi -#define CGEMM3M_ITCOPYB gotoblas -> cgemm3m_itcopyb -#define CGEMM3M_ITCOPYR gotoblas -> cgemm3m_itcopyr -#define CGEMM3M_ITCOPYI gotoblas -> cgemm3m_itcopyi - -#define CSYMM3M_ILCOPYB gotoblas -> csymm3m_ilcopyb -#define CSYMM3M_IUCOPYB gotoblas -> csymm3m_iucopyb -#define CSYMM3M_ILCOPYR gotoblas -> csymm3m_ilcopyr -#define CSYMM3M_IUCOPYR gotoblas -> csymm3m_iucopyr -#define CSYMM3M_ILCOPYI gotoblas -> csymm3m_ilcopyi -#define CSYMM3M_IUCOPYI gotoblas -> csymm3m_iucopyi - -#define CSYMM3M_OLCOPYB gotoblas -> csymm3m_olcopyb -#define CSYMM3M_OUCOPYB gotoblas -> csymm3m_oucopyb -#define CSYMM3M_OLCOPYR gotoblas -> csymm3m_olcopyr -#define CSYMM3M_OUCOPYR gotoblas -> csymm3m_oucopyr -#define CSYMM3M_OLCOPYI gotoblas -> csymm3m_olcopyi -#define CSYMM3M_OUCOPYI gotoblas -> csymm3m_oucopyi - -#define CHEMM3M_ILCOPYB gotoblas -> chemm3m_ilcopyb -#define CHEMM3M_IUCOPYB gotoblas -> chemm3m_iucopyb -#define CHEMM3M_ILCOPYR gotoblas -> chemm3m_ilcopyr -#define CHEMM3M_IUCOPYR gotoblas -> chemm3m_iucopyr -#define CHEMM3M_ILCOPYI gotoblas -> chemm3m_ilcopyi -#define CHEMM3M_IUCOPYI gotoblas -> chemm3m_iucopyi - -#define CHEMM3M_OLCOPYB gotoblas -> chemm3m_olcopyb -#define CHEMM3M_OUCOPYB gotoblas -> chemm3m_oucopyb -#define CHEMM3M_OLCOPYR gotoblas -> chemm3m_olcopyr -#define CHEMM3M_OUCOPYR gotoblas -> chemm3m_oucopyr -#define CHEMM3M_OLCOPYI gotoblas -> chemm3m_olcopyi -#define CHEMM3M_OUCOPYI gotoblas -> chemm3m_oucopyi - -#define CGEMM3M_KERNEL gotoblas -> cgemm3m_kernel - -#define CNEG_TCOPY gotoblas -> cneg_tcopy -#define CLASWP_NCOPY gotoblas -> claswp_ncopy - -#define CAXPBY_K gotoblas -> caxpby_k - -#define COMATCOPY_K_CN gotoblas -> comatcopy_k_cn -#define COMATCOPY_K_RN gotoblas -> comatcopy_k_rn -#define COMATCOPY_K_CT gotoblas -> comatcopy_k_ct -#define COMATCOPY_K_RT gotoblas -> comatcopy_k_rt -#define COMATCOPY_K_CNC gotoblas -> comatcopy_k_cnc -#define COMATCOPY_K_RNC gotoblas -> comatcopy_k_rnc -#define COMATCOPY_K_CTC gotoblas -> comatcopy_k_ctc -#define COMATCOPY_K_RTC gotoblas -> comatcopy_k_rtc - -#define CIMATCOPY_K_CN gotoblas -> cimatcopy_k_cn -#define CIMATCOPY_K_RN gotoblas -> cimatcopy_k_rn -#define CIMATCOPY_K_CT gotoblas -> cimatcopy_k_ct -#define CIMATCOPY_K_RT gotoblas -> cimatcopy_k_rt -#define CIMATCOPY_K_CNC gotoblas -> cimatcopy_k_cnc -#define CIMATCOPY_K_RNC gotoblas -> cimatcopy_k_rnc -#define CIMATCOPY_K_CTC gotoblas -> cimatcopy_k_ctc -#define CIMATCOPY_K_RTC gotoblas -> cimatcopy_k_rtc - -#define CGEADD_K gotoblas -> cgeadd_k - -#define CGEMM_SMALL_MATRIX_PERMIT gotoblas -> cgemm_small_matrix_permit +#define CGEMM_ONCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_oncopy +#define CGEMM_OTCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_otcopy +#define CGEMM_INCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_incopy +#define CGEMM_ITCOPY OPENBLAS_DISPATCH(cgemm) -> cgemm_itcopy + +#define CTRMM_OUNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ounucopy +#define CTRMM_OUTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_outucopy +#define CTRMM_OLNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_olnucopy +#define CTRMM_OLTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_oltucopy +#define CTRSM_OUNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ounucopy +#define CTRSM_OUTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_outucopy +#define CTRSM_OLNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_olnucopy +#define CTRSM_OLTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_oltucopy + +#define CTRMM_IUNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iunucopy +#define CTRMM_IUTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iutucopy +#define CTRMM_ILNUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ilnucopy +#define CTRMM_ILTUCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iltucopy +#define CTRSM_IUNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iunucopy +#define CTRSM_IUTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iutucopy +#define CTRSM_ILNUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ilnucopy +#define CTRSM_ILTUCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iltucopy + +#define CTRMM_OUNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ounncopy +#define CTRMM_OUTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_outncopy +#define CTRMM_OLNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_olnncopy +#define CTRMM_OLTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_oltncopy +#define CTRSM_OUNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ounncopy +#define CTRSM_OUTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_outncopy +#define CTRSM_OLNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_olnncopy +#define CTRSM_OLTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_oltncopy + +#define CTRMM_IUNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iunncopy +#define CTRMM_IUTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iutncopy +#define CTRMM_ILNNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_ilnncopy +#define CTRMM_ILTNCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_iltncopy +#define CSYMM_INCOPY OPENBLAS_DISPATCH(csymm) -> csymm_incopy +#define CSYMM_ITCOPY OPENBLAS_DISPATCH(csymm) -> csymm_itcopy +#define CTRMM_INCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_incopy +#define CTRMM_ITCOPY OPENBLAS_DISPATCH(ctrmm) -> ctrmm_itcopy + +#define CTRSM_IUNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iunncopy +#define CTRSM_IUTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iutncopy +#define CTRSM_ILNNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_ilnncopy +#define CTRSM_ILTNCOPY OPENBLAS_DISPATCH(ctrsm) -> ctrsm_iltncopy + +#define SME_CGEMM_KERNEL OPENBLAS_DISPATCH(cgemm) -> sme_cgemm_kernel +#define CGEMM_BETA OPENBLAS_DISPATCH(cgemm) -> cgemm_beta +#define CGEMM_KERNEL_N OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_n +#define CGEMM_KERNEL_L OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_l +#define CGEMM_KERNEL_R OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_r +#define CGEMM_KERNEL_B OPENBLAS_DISPATCH(cgemm) -> cgemm_kernel_b + +#define CSYMM_KERNEL_N OPENBLAS_DISPATCH(csymm) -> csymm_kernel_n +#define CSYMM_KERNEL_L OPENBLAS_DISPATCH(csymm) -> csymm_kernel_l +#define CSYMM_KERNEL_R OPENBLAS_DISPATCH(csymm) -> csymm_kernel_r +#define CSYMM_KERNEL_B OPENBLAS_DISPATCH(csymm) -> csymm_kernel_b +#define CTRMM_GEMM_KERNEL_N OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_n +#define CTRMM_GEMM_KERNEL_L OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_l +#define CTRMM_GEMM_KERNEL_R OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_r +#define CTRMM_GEMM_KERNEL_B OPENBLAS_DISPATCH(ctrmm) -> ctrmm_gemm_kernel_b + +#define CTRMM_KERNEL_LN OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LN +#define CTRMM_KERNEL_LT OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LT +#define CTRMM_KERNEL_LR OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LR +#define CTRMM_KERNEL_LC OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_LC +#define CTRMM_KERNEL_RN OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RN +#define CTRMM_KERNEL_RT OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RT +#define CTRMM_KERNEL_RR OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RR +#define CTRMM_KERNEL_RC OPENBLAS_DISPATCH(ctrmm) -> ctrmm_kernel_RC + +#define CTRSM_KERNEL_LN OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LN +#define CTRSM_KERNEL_LT OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LT +#define CTRSM_KERNEL_LR OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LR +#define CTRSM_KERNEL_LC OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_LC +#define CTRSM_KERNEL_RN OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RN +#define CTRSM_KERNEL_RT OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RT +#define CTRSM_KERNEL_RR OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RR +#define CTRSM_KERNEL_RC OPENBLAS_DISPATCH(ctrsm) -> ctrsm_kernel_RC + +#define CSYMM_IUTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_iutcopy +#define CSYMM_ILTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_iltcopy +#define CSYMM_OUTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_outcopy +#define CSYMM_OLTCOPY OPENBLAS_DISPATCH(csymm) -> csymm_oltcopy + +#define CHEMM_OUTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_outcopy +#define CHEMM_OLTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_oltcopy +#define CHEMM_IUTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_iutcopy +#define CHEMM_ILTCOPY OPENBLAS_DISPATCH(chemm) -> chemm_iltcopy + +#define CGEMM3M_ONCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyb +#define CGEMM3M_ONCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyr +#define CGEMM3M_ONCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_oncopyi +#define CGEMM3M_OTCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyb +#define CGEMM3M_OTCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyr +#define CGEMM3M_OTCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_otcopyi + +#define CGEMM3M_INCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyb +#define CGEMM3M_INCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyr +#define CGEMM3M_INCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_incopyi +#define CGEMM3M_ITCOPYB OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyb +#define CGEMM3M_ITCOPYR OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyr +#define CGEMM3M_ITCOPYI OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_itcopyi + +#define CSYMM3M_ILCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyb +#define CSYMM3M_IUCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyb +#define CSYMM3M_ILCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyr +#define CSYMM3M_IUCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyr +#define CSYMM3M_ILCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_ilcopyi +#define CSYMM3M_IUCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_iucopyi + +#define CSYMM3M_OLCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyb +#define CSYMM3M_OUCOPYB OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyb +#define CSYMM3M_OLCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyr +#define CSYMM3M_OUCOPYR OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyr +#define CSYMM3M_OLCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_olcopyi +#define CSYMM3M_OUCOPYI OPENBLAS_DISPATCH(csymm3m) -> csymm3m_oucopyi + +#define CHEMM3M_ILCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyb +#define CHEMM3M_IUCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyb +#define CHEMM3M_ILCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyr +#define CHEMM3M_IUCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyr +#define CHEMM3M_ILCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_ilcopyi +#define CHEMM3M_IUCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_iucopyi + +#define CHEMM3M_OLCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyb +#define CHEMM3M_OUCOPYB OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyb +#define CHEMM3M_OLCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyr +#define CHEMM3M_OUCOPYR OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyr +#define CHEMM3M_OLCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_olcopyi +#define CHEMM3M_OUCOPYI OPENBLAS_DISPATCH(chemm3m) -> chemm3m_oucopyi + +#define CGEMM3M_KERNEL OPENBLAS_DISPATCH(cgemm3m) -> cgemm3m_kernel + +#define CNEG_TCOPY OPENBLAS_DISPATCH(cneg) -> cneg_tcopy +#define CLASWP_NCOPY OPENBLAS_DISPATCH(claswp) -> claswp_ncopy + +#define CAXPBY_K OPENBLAS_DISPATCH(caxpby) -> caxpby_k + +#define COMATCOPY_K_CN OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_cn +#define COMATCOPY_K_RN OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rn +#define COMATCOPY_K_CT OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_ct +#define COMATCOPY_K_RT OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rt +#define COMATCOPY_K_CNC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_cnc +#define COMATCOPY_K_RNC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rnc +#define COMATCOPY_K_CTC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_ctc +#define COMATCOPY_K_RTC OPENBLAS_DISPATCH(comatcopy) -> comatcopy_k_rtc + +#define CIMATCOPY_K_CN OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_cn +#define CIMATCOPY_K_RN OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rn +#define CIMATCOPY_K_CT OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_ct +#define CIMATCOPY_K_RT OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rt +#define CIMATCOPY_K_CNC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_cnc +#define CIMATCOPY_K_RNC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rnc +#define CIMATCOPY_K_CTC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_ctc +#define CIMATCOPY_K_RTC OPENBLAS_DISPATCH(cimatcopy) -> cimatcopy_k_rtc + +#define CGEADD_K OPENBLAS_DISPATCH(cgeadd) -> cgeadd_k + +#define CGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(cgemm) -> cgemm_small_matrix_permit #endif -#define CGEMM_SMALL_KERNEL_BASE FUNC_BASE(cgemm_small_kernel_nn) -#define CGEMM_SMALL_KERNEL_NN FUNC_OFFSET(cgemm_small_kernel_nn) -#define CGEMM_SMALL_KERNEL_NT FUNC_OFFSET(cgemm_small_kernel_nt) -#define CGEMM_SMALL_KERNEL_NR FUNC_OFFSET(cgemm_small_kernel_nr) -#define CGEMM_SMALL_KERNEL_NC FUNC_OFFSET(cgemm_small_kernel_nc) - -#define CGEMM_SMALL_KERNEL_TN FUNC_OFFSET(cgemm_small_kernel_tn) -#define CGEMM_SMALL_KERNEL_TT FUNC_OFFSET(cgemm_small_kernel_tt) -#define CGEMM_SMALL_KERNEL_TR FUNC_OFFSET(cgemm_small_kernel_tr) -#define CGEMM_SMALL_KERNEL_TC FUNC_OFFSET(cgemm_small_kernel_tc) - -#define CGEMM_SMALL_KERNEL_RN FUNC_OFFSET(cgemm_small_kernel_rn) -#define CGEMM_SMALL_KERNEL_RT FUNC_OFFSET(cgemm_small_kernel_rt) -#define CGEMM_SMALL_KERNEL_RR FUNC_OFFSET(cgemm_small_kernel_rr) -#define CGEMM_SMALL_KERNEL_RC FUNC_OFFSET(cgemm_small_kernel_rc) - -#define CGEMM_SMALL_KERNEL_CN FUNC_OFFSET(cgemm_small_kernel_cn) -#define CGEMM_SMALL_KERNEL_CT FUNC_OFFSET(cgemm_small_kernel_ct) -#define CGEMM_SMALL_KERNEL_CR FUNC_OFFSET(cgemm_small_kernel_cr) -#define CGEMM_SMALL_KERNEL_CC FUNC_OFFSET(cgemm_small_kernel_cc) - -#define CGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(cgemm_small_kernel_b0_nn) -#define CGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(cgemm_small_kernel_b0_nt) -#define CGEMM_SMALL_KERNEL_B0_NR FUNC_OFFSET(cgemm_small_kernel_b0_nr) -#define CGEMM_SMALL_KERNEL_B0_NC FUNC_OFFSET(cgemm_small_kernel_b0_nc) - -#define CGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(cgemm_small_kernel_b0_tn) -#define CGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(cgemm_small_kernel_b0_tt) -#define CGEMM_SMALL_KERNEL_B0_TR FUNC_OFFSET(cgemm_small_kernel_b0_tr) -#define CGEMM_SMALL_KERNEL_B0_TC FUNC_OFFSET(cgemm_small_kernel_b0_tc) - -#define CGEMM_SMALL_KERNEL_B0_RN FUNC_OFFSET(cgemm_small_kernel_b0_rn) -#define CGEMM_SMALL_KERNEL_B0_RT FUNC_OFFSET(cgemm_small_kernel_b0_rt) -#define CGEMM_SMALL_KERNEL_B0_RR FUNC_OFFSET(cgemm_small_kernel_b0_rr) -#define CGEMM_SMALL_KERNEL_B0_RC FUNC_OFFSET(cgemm_small_kernel_b0_rc) - -#define CGEMM_SMALL_KERNEL_B0_CN FUNC_OFFSET(cgemm_small_kernel_b0_cn) -#define CGEMM_SMALL_KERNEL_B0_CT FUNC_OFFSET(cgemm_small_kernel_b0_ct) -#define CGEMM_SMALL_KERNEL_B0_CR FUNC_OFFSET(cgemm_small_kernel_b0_cr) -#define CGEMM_SMALL_KERNEL_B0_CC FUNC_OFFSET(cgemm_small_kernel_b0_cc) +#define CGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(cgemm) +#define CGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nn) +#define CGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nt) +#define CGEMM_SMALL_KERNEL_NR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nr) +#define CGEMM_SMALL_KERNEL_NC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_nc) + +#define CGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tn) +#define CGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tt) +#define CGEMM_SMALL_KERNEL_TR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tr) +#define CGEMM_SMALL_KERNEL_TC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_tc) + +#define CGEMM_SMALL_KERNEL_RN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rn) +#define CGEMM_SMALL_KERNEL_RT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rt) +#define CGEMM_SMALL_KERNEL_RR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rr) +#define CGEMM_SMALL_KERNEL_RC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_rc) + +#define CGEMM_SMALL_KERNEL_CN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cn) +#define CGEMM_SMALL_KERNEL_CT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_ct) +#define CGEMM_SMALL_KERNEL_CR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cr) +#define CGEMM_SMALL_KERNEL_CC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_cc) + +#define CGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nn) +#define CGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nt) +#define CGEMM_SMALL_KERNEL_B0_NR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nr) +#define CGEMM_SMALL_KERNEL_B0_NC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_nc) + +#define CGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tn) +#define CGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tt) +#define CGEMM_SMALL_KERNEL_B0_TR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tr) +#define CGEMM_SMALL_KERNEL_B0_TC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_tc) + +#define CGEMM_SMALL_KERNEL_B0_RN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rn) +#define CGEMM_SMALL_KERNEL_B0_RT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rt) +#define CGEMM_SMALL_KERNEL_B0_RR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rr) +#define CGEMM_SMALL_KERNEL_B0_RC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_rc) + +#define CGEMM_SMALL_KERNEL_B0_CN OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cn) +#define CGEMM_SMALL_KERNEL_B0_CT OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_ct) +#define CGEMM_SMALL_KERNEL_B0_CR OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cr) +#define CGEMM_SMALL_KERNEL_B0_CC OPENBLAS_DISPATCH_OFFSET(cgemm, cgemm_small_kernel_b0_cc) #define CGEMM_NN cgemm_nn diff --git a/common_d.h b/common_d.h index 6eef671401..cefd1fc6dc 100644 --- a/common_d.h +++ b/common_d.h @@ -209,151 +209,151 @@ #else -#define DAMAX_K gotoblas -> damax_k -#define DAMIN_K gotoblas -> damin_k -#define DMAX_K gotoblas -> dmax_k -#define DMIN_K gotoblas -> dmin_k -#define IDAMAX_K gotoblas -> idamax_k -#define IDAMIN_K gotoblas -> idamin_k -#define IDMAX_K gotoblas -> idmax_k -#define IDMIN_K gotoblas -> idmin_k -#define DASUM_K gotoblas -> dasum_k -#define DAXPYU_K gotoblas -> daxpy_k -#define DAXPYC_K gotoblas -> daxpy_k -#define DCOPY_K gotoblas -> dcopy_k -#define DDOTU_K gotoblas -> ddot_k -#define DDOTC_K gotoblas -> ddot_k -#define DNRM2_K gotoblas -> dnrm2_k -#define DSCAL_K gotoblas -> dscal_k -#define DSUM_K gotoblas -> dsum_k -#define DSWAP_K gotoblas -> dswap_k -#define DROT_K gotoblas -> drot_k -#define DROTM_K gotoblas -> drotm_k - -#define DGEMV_N gotoblas -> dgemv_n -#define DGEMV_T gotoblas -> dgemv_t -#define DGEMV_R gotoblas -> dgemv_n -#define DGEMV_C gotoblas -> dgemv_t -#define DGEMV_O gotoblas -> dgemv_n -#define DGEMV_U gotoblas -> dgemv_t -#define DGEMV_S gotoblas -> dgemv_n -#define DGEMV_D gotoblas -> dgemv_t - -#define DGERU_K gotoblas -> dger_k -#define DGERC_K gotoblas -> dger_k -#define DGERV_K gotoblas -> dger_k -#define DGERD_K gotoblas -> dger_k - -#define DSYMV_U gotoblas -> dsymv_U -#define DSYMV_L gotoblas -> dsymv_L +#define DAMAX_K OPENBLAS_DISPATCH(damax) -> damax_k +#define DAMIN_K OPENBLAS_DISPATCH(damin) -> damin_k +#define DMAX_K OPENBLAS_DISPATCH(dmax) -> dmax_k +#define DMIN_K OPENBLAS_DISPATCH(dmin) -> dmin_k +#define IDAMAX_K OPENBLAS_DISPATCH(idamax) -> idamax_k +#define IDAMIN_K OPENBLAS_DISPATCH(idamin) -> idamin_k +#define IDMAX_K OPENBLAS_DISPATCH(idmax) -> idmax_k +#define IDMIN_K OPENBLAS_DISPATCH(idmin) -> idmin_k +#define DASUM_K OPENBLAS_DISPATCH(dasum) -> dasum_k +#define DAXPYU_K OPENBLAS_DISPATCH(daxpy) -> daxpy_k +#define DAXPYC_K OPENBLAS_DISPATCH(daxpy) -> daxpy_k +#define DCOPY_K OPENBLAS_DISPATCH(dcopy) -> dcopy_k +#define DDOTU_K OPENBLAS_DISPATCH(ddot) -> ddot_k +#define DDOTC_K OPENBLAS_DISPATCH(ddot) -> ddot_k +#define DNRM2_K OPENBLAS_DISPATCH(dnrm2) -> dnrm2_k +#define DSCAL_K OPENBLAS_DISPATCH(dscal) -> dscal_k +#define DSUM_K OPENBLAS_DISPATCH(dsum) -> dsum_k +#define DSWAP_K OPENBLAS_DISPATCH(dswap) -> dswap_k +#define DROT_K OPENBLAS_DISPATCH(drot) -> drot_k +#define DROTM_K OPENBLAS_DISPATCH(drotm) -> drotm_k + +#define DGEMV_N OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_T OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_R OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_C OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_O OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_U OPENBLAS_DISPATCH(dgemv) -> dgemv_t +#define DGEMV_S OPENBLAS_DISPATCH(dgemv) -> dgemv_n +#define DGEMV_D OPENBLAS_DISPATCH(dgemv) -> dgemv_t + +#define DGERU_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERC_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERV_K OPENBLAS_DISPATCH(dger) -> dger_k +#define DGERD_K OPENBLAS_DISPATCH(dger) -> dger_k + +#define DSYMV_U OPENBLAS_DISPATCH(dsymv) -> dsymv_U +#define DSYMV_L OPENBLAS_DISPATCH(dsymv) -> dsymv_L #define DSYMV_THREAD_U dsymv_thread_U #define DSYMV_THREAD_L dsymv_thread_L -#define DGEMM_ONCOPY gotoblas -> dgemm_oncopy -#define DGEMM_OTCOPY gotoblas -> dgemm_otcopy -#define DGEMM_INCOPY gotoblas -> dgemm_incopy -#define DGEMM_ITCOPY gotoblas -> dgemm_itcopy - -#define DTRMM_OUNUCOPY gotoblas -> dtrmm_ounucopy -#define DTRMM_OUTUCOPY gotoblas -> dtrmm_outucopy -#define DTRMM_OLNUCOPY gotoblas -> dtrmm_olnucopy -#define DTRMM_OLTUCOPY gotoblas -> dtrmm_oltucopy -#define DTRSM_OUNUCOPY gotoblas -> dtrsm_ounucopy -#define DTRSM_OUTUCOPY gotoblas -> dtrsm_outucopy -#define DTRSM_OLNUCOPY gotoblas -> dtrsm_olnucopy -#define DTRSM_OLTUCOPY gotoblas -> dtrsm_oltucopy - -#define DTRMM_IUNUCOPY gotoblas -> dtrmm_iunucopy -#define DTRMM_IUTUCOPY gotoblas -> dtrmm_iutucopy -#define DTRMM_ILNUCOPY gotoblas -> dtrmm_ilnucopy -#define DTRMM_ILTUCOPY gotoblas -> dtrmm_iltucopy -#define DTRSM_IUNUCOPY gotoblas -> dtrsm_iunucopy -#define DTRSM_IUTUCOPY gotoblas -> dtrsm_iutucopy -#define DTRSM_ILNUCOPY gotoblas -> dtrsm_ilnucopy -#define DTRSM_ILTUCOPY gotoblas -> dtrsm_iltucopy - -#define DTRMM_OUNNCOPY gotoblas -> dtrmm_ounncopy -#define DTRMM_OUTNCOPY gotoblas -> dtrmm_outncopy -#define DTRMM_OLNNCOPY gotoblas -> dtrmm_olnncopy -#define DTRMM_OLTNCOPY gotoblas -> dtrmm_oltncopy -#define DTRSM_OUNNCOPY gotoblas -> dtrsm_ounncopy -#define DTRSM_OUTNCOPY gotoblas -> dtrsm_outncopy -#define DTRSM_OLNNCOPY gotoblas -> dtrsm_olnncopy -#define DTRSM_OLTNCOPY gotoblas -> dtrsm_oltncopy - -#define DTRMM_IUNNCOPY gotoblas -> dtrmm_iunncopy -#define DTRMM_IUTNCOPY gotoblas -> dtrmm_iutncopy -#define DTRMM_ILNNCOPY gotoblas -> dtrmm_ilnncopy -#define DTRMM_ILTNCOPY gotoblas -> dtrmm_iltncopy -#define DSYMM_INCOPY gotoblas -> dsymm_incopy -#define DSYMM_ITCOPY gotoblas -> dsymm_itcopy -#define DTRMM_INCOPY gotoblas -> dtrmm_incopy -#define DTRMM_ITCOPY gotoblas -> dtrmm_itcopy - -#define DTRSM_IUNNCOPY gotoblas -> dtrsm_iunncopy -#define DTRSM_IUTNCOPY gotoblas -> dtrsm_iutncopy -#define DTRSM_ILNNCOPY gotoblas -> dtrsm_ilnncopy -#define DTRSM_ILTNCOPY gotoblas -> dtrsm_iltncopy - -#define DGEMM_BETA gotoblas -> dgemm_beta -#define DGEMM_KERNEL gotoblas -> dgemm_kernel -#define SME_DGEMM_KERNEL gotoblas -> sme_dgemm_kernel -#define DSYMM_KERNEL gotoblas -> dsymm_kernel -#define DTRMM_GEMM_KERNEL gotoblas -> dtrmm_gemm_kernel - -#define DTRMM_KERNEL_LN gotoblas -> dtrmm_kernel_LN -#define DTRMM_KERNEL_LT gotoblas -> dtrmm_kernel_LT -#define DTRMM_KERNEL_LR gotoblas -> dtrmm_kernel_LN -#define DTRMM_KERNEL_LC gotoblas -> dtrmm_kernel_LT -#define DTRMM_KERNEL_RN gotoblas -> dtrmm_kernel_RN -#define DTRMM_KERNEL_RT gotoblas -> dtrmm_kernel_RT -#define DTRMM_KERNEL_RR gotoblas -> dtrmm_kernel_RN -#define DTRMM_KERNEL_RC gotoblas -> dtrmm_kernel_RT - -#define DTRSM_KERNEL_LN gotoblas -> dtrsm_kernel_LN -#define DTRSM_KERNEL_LT gotoblas -> dtrsm_kernel_LT -#define DTRSM_KERNEL_LR gotoblas -> dtrsm_kernel_LN -#define DTRSM_KERNEL_LC gotoblas -> dtrsm_kernel_LT -#define DTRSM_KERNEL_RN gotoblas -> dtrsm_kernel_RN -#define DTRSM_KERNEL_RT gotoblas -> dtrsm_kernel_RT -#define DTRSM_KERNEL_RR gotoblas -> dtrsm_kernel_RN -#define DTRSM_KERNEL_RC gotoblas -> dtrsm_kernel_RT - -#define DSYMM_IUTCOPY gotoblas -> dsymm_iutcopy -#define DSYMM_ILTCOPY gotoblas -> dsymm_iltcopy -#define DSYMM_OUTCOPY gotoblas -> dsymm_outcopy -#define DSYMM_OLTCOPY gotoblas -> dsymm_oltcopy - -#define DNEG_TCOPY gotoblas -> dneg_tcopy -#define DLASWP_NCOPY gotoblas -> dlaswp_ncopy - -#define DAXPBY_K gotoblas -> daxpby_k -#define DOMATCOPY_K_CN gotoblas -> domatcopy_k_cn -#define DOMATCOPY_K_RN gotoblas -> domatcopy_k_rn -#define DOMATCOPY_K_CT gotoblas -> domatcopy_k_ct -#define DOMATCOPY_K_RT gotoblas -> domatcopy_k_rt -#define DIMATCOPY_K_CN gotoblas -> dimatcopy_k_cn -#define DIMATCOPY_K_RN gotoblas -> dimatcopy_k_rn -#define DIMATCOPY_K_CT gotoblas -> dimatcopy_k_ct -#define DIMATCOPY_K_RT gotoblas -> dimatcopy_k_rt - -#define DGEADD_K gotoblas -> dgeadd_k - -#define DGEMM_SMALL_MATRIX_PERMIT gotoblas -> dgemm_small_matrix_permit +#define DGEMM_ONCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_oncopy +#define DGEMM_OTCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_otcopy +#define DGEMM_INCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_incopy +#define DGEMM_ITCOPY OPENBLAS_DISPATCH(dgemm) -> dgemm_itcopy + +#define DTRMM_OUNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ounucopy +#define DTRMM_OUTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_outucopy +#define DTRMM_OLNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_olnucopy +#define DTRMM_OLTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_oltucopy +#define DTRSM_OUNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ounucopy +#define DTRSM_OUTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_outucopy +#define DTRSM_OLNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_olnucopy +#define DTRSM_OLTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_oltucopy + +#define DTRMM_IUNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iunucopy +#define DTRMM_IUTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iutucopy +#define DTRMM_ILNUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ilnucopy +#define DTRMM_ILTUCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iltucopy +#define DTRSM_IUNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iunucopy +#define DTRSM_IUTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iutucopy +#define DTRSM_ILNUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ilnucopy +#define DTRSM_ILTUCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iltucopy + +#define DTRMM_OUNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ounncopy +#define DTRMM_OUTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_outncopy +#define DTRMM_OLNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_olnncopy +#define DTRMM_OLTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_oltncopy +#define DTRSM_OUNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ounncopy +#define DTRSM_OUTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_outncopy +#define DTRSM_OLNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_olnncopy +#define DTRSM_OLTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_oltncopy + +#define DTRMM_IUNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iunncopy +#define DTRMM_IUTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iutncopy +#define DTRMM_ILNNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_ilnncopy +#define DTRMM_ILTNCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_iltncopy +#define DSYMM_INCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_incopy +#define DSYMM_ITCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_itcopy +#define DTRMM_INCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_incopy +#define DTRMM_ITCOPY OPENBLAS_DISPATCH(dtrmm) -> dtrmm_itcopy + +#define DTRSM_IUNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iunncopy +#define DTRSM_IUTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iutncopy +#define DTRSM_ILNNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_ilnncopy +#define DTRSM_ILTNCOPY OPENBLAS_DISPATCH(dtrsm) -> dtrsm_iltncopy + +#define DGEMM_BETA OPENBLAS_DISPATCH(dgemm) -> dgemm_beta +#define DGEMM_KERNEL OPENBLAS_DISPATCH(dgemm) -> dgemm_kernel +#define SME_DGEMM_KERNEL OPENBLAS_DISPATCH(dgemm) -> sme_dgemm_kernel +#define DSYMM_KERNEL OPENBLAS_DISPATCH(dsymm) -> dsymm_kernel +#define DTRMM_GEMM_KERNEL OPENBLAS_DISPATCH(dtrmm) -> dtrmm_gemm_kernel + +#define DTRMM_KERNEL_LN OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LN +#define DTRMM_KERNEL_LT OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LT +#define DTRMM_KERNEL_LR OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LN +#define DTRMM_KERNEL_LC OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_LT +#define DTRMM_KERNEL_RN OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RN +#define DTRMM_KERNEL_RT OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RT +#define DTRMM_KERNEL_RR OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RN +#define DTRMM_KERNEL_RC OPENBLAS_DISPATCH(dtrmm) -> dtrmm_kernel_RT + +#define DTRSM_KERNEL_LN OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LN +#define DTRSM_KERNEL_LT OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LT +#define DTRSM_KERNEL_LR OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LN +#define DTRSM_KERNEL_LC OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_LT +#define DTRSM_KERNEL_RN OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RN +#define DTRSM_KERNEL_RT OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RT +#define DTRSM_KERNEL_RR OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RN +#define DTRSM_KERNEL_RC OPENBLAS_DISPATCH(dtrsm) -> dtrsm_kernel_RT + +#define DSYMM_IUTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_iutcopy +#define DSYMM_ILTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_iltcopy +#define DSYMM_OUTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_outcopy +#define DSYMM_OLTCOPY OPENBLAS_DISPATCH(dsymm) -> dsymm_oltcopy + +#define DNEG_TCOPY OPENBLAS_DISPATCH(dneg) -> dneg_tcopy +#define DLASWP_NCOPY OPENBLAS_DISPATCH(dlaswp) -> dlaswp_ncopy + +#define DAXPBY_K OPENBLAS_DISPATCH(daxpby) -> daxpby_k +#define DOMATCOPY_K_CN OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_cn +#define DOMATCOPY_K_RN OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_rn +#define DOMATCOPY_K_CT OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_ct +#define DOMATCOPY_K_RT OPENBLAS_DISPATCH(domatcopy) -> domatcopy_k_rt +#define DIMATCOPY_K_CN OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_cn +#define DIMATCOPY_K_RN OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_rn +#define DIMATCOPY_K_CT OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_ct +#define DIMATCOPY_K_RT OPENBLAS_DISPATCH(dimatcopy) -> dimatcopy_k_rt + +#define DGEADD_K OPENBLAS_DISPATCH(dgeadd) -> dgeadd_k + +#define DGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(dgemm) -> dgemm_small_matrix_permit #endif -#define DGEMM_SMALL_KERNEL_BASE FUNC_BASE(dgemm_small_kernel_nn) -#define DGEMM_SMALL_KERNEL_NN FUNC_OFFSET(dgemm_small_kernel_nn) -#define DGEMM_SMALL_KERNEL_NT FUNC_OFFSET(dgemm_small_kernel_nt) -#define DGEMM_SMALL_KERNEL_TN FUNC_OFFSET(dgemm_small_kernel_tn) -#define DGEMM_SMALL_KERNEL_TT FUNC_OFFSET(dgemm_small_kernel_tt) +#define DGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(dgemm) +#define DGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_nn) +#define DGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_nt) +#define DGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_tn) +#define DGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_tt) -#define DGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(dgemm_small_kernel_b0_nn) -#define DGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(dgemm_small_kernel_b0_nt) -#define DGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(dgemm_small_kernel_b0_tn) -#define DGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(dgemm_small_kernel_b0_tt) +#define DGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_nn) +#define DGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_nt) +#define DGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_tn) +#define DGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(dgemm, dgemm_small_kernel_b0_tt) #define DGEMM_NN dgemm_nn diff --git a/common_param.h b/common_param.h index 0bc504bdbb..71d8a0dc52 100644 --- a/common_param.h +++ b/common_param.h @@ -66,90 +66,313 @@ typedef struct { #if BUILD_HFLOAT16 == 1 int shgemm_p, shgemm_q, shgemm_r; int shgemm_unroll_m, shgemm_unroll_n, shgemm_unroll_mn; +#endif +#if BUILD_BFLOAT16 == 1 + int bgemm_p, bgemm_q, bgemm_r; + int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; + int bgemm_align_k; + int sbgemm_p, sbgemm_q, sbgemm_r; + int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; + int sbgemm_align_k; + int need_amxtile_permission; // 0 default, 1 for device support amx. +#endif +#if (BUILD_SINGLE == 1) || (BUILD_DOUBLE == 1) || (BUILD_COMPLEX == 1) || (BUILD_COMPLEX16 == 1) + int sgemm_p, sgemm_q, sgemm_r; + int sgemm_unroll_m, sgemm_unroll_n, sgemm_unroll_mn; +#endif + int exclusive_cache; +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + int dgemm_p, dgemm_q, dgemm_r; + int dgemm_unroll_m, dgemm_unroll_n, dgemm_unroll_mn; +#endif +#ifdef EXPRECISION + int qgemm_p, qgemm_q, qgemm_r; + int qgemm_unroll_m, qgemm_unroll_n, qgemm_unroll_mn; +#endif +#if (BUILD_COMPLEX==1) + int cgemm_p, cgemm_q, cgemm_r; + int cgemm_unroll_m, cgemm_unroll_n, cgemm_unroll_mn; + int cgemm3m_p, cgemm3m_q, cgemm3m_r; + int cgemm3m_unroll_m, cgemm3m_unroll_n, cgemm3m_unroll_mn; +#endif +#if (BUILD_COMPLEX16 == 1) + int zgemm_p, zgemm_q, zgemm_r; + int zgemm_unroll_m, zgemm_unroll_n, zgemm_unroll_mn; + int zgemm3m_p, zgemm3m_q, zgemm3m_r; + int zgemm3m_unroll_m, zgemm3m_unroll_n, zgemm3m_unroll_mn; +#endif +#ifdef EXPRECISION + int xgemm_p, xgemm_q, xgemm_r; + int xgemm_unroll_m, xgemm_unroll_n, xgemm_unroll_mn; + int xgemm3m_p, xgemm3m_q, xgemm3m_r; + int xgemm3m_unroll_m, xgemm3m_unroll_n, xgemm3m_unroll_mn; +#endif + void (*init)(void); + int snum_opt, dnum_opt, qnum_opt; +} gotoblas_t; + +extern gotoblas_t *gotoblas; +#if BUILD_HFLOAT16 == 1 +typedef struct { int (*shgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, hfloat16 *, float *, BLASLONG); int (*shgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float *, BLASLONG); - int (*shgemm_incopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_itcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_oncopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); +} openblas_shgemm_dispatch_t; +extern const openblas_shgemm_dispatch_t *const openblas_shgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_HFLOAT16 == 1 +typedef struct { int (*shgemv_n) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG); int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG); +} openblas_shgemv_dispatch_t; +extern const openblas_shgemv_dispatch_t *const openblas_shgemv_dispatch[OPENBLAS_NUM_CORES]; #endif - #if BUILD_BFLOAT16 == 1 - int bgemm_p, bgemm_q, bgemm_r; - int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; - int bgemm_align_k; - - int sbgemm_p, sbgemm_q, sbgemm_r; - int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; - int sbgemm_align_k; - int need_amxtile_permission; // 0 default, 1 for device support amx. - +typedef struct { void (*sbstobf16_k) (BLASLONG, float *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbstobf16_dispatch_t; +extern const openblas_sbstobf16_dispatch_t *const openblas_sbstobf16_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*sbdtobf16_k) (BLASLONG, double *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbdtobf16_dispatch_t; +extern const openblas_sbdtobf16_dispatch_t *const openblas_sbdtobf16_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*sbf16tos_k) (BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); +} openblas_sbf16tos_dispatch_t; +extern const openblas_sbf16tos_dispatch_t *const openblas_sbf16tos_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { void (*dbf16tod_k) (BLASLONG, bfloat16 *, BLASLONG, double *, BLASLONG); +} openblas_dbf16tod_dispatch_t; +extern const openblas_dbf16tod_dispatch_t *const openblas_dbf16tod_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbamax_k) (BLASLONG, float *, BLASLONG); +} openblas_sbamax_dispatch_t; +extern const openblas_sbamax_dispatch_t *const openblas_sbamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbamin_k) (BLASLONG, float *, BLASLONG); +} openblas_sbamin_dispatch_t; +extern const openblas_sbamin_dispatch_t *const openblas_sbamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbmax_k) (BLASLONG, float *, BLASLONG); +} openblas_sbmax_dispatch_t; +extern const openblas_sbmax_dispatch_t *const openblas_sbmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbmin_k) (BLASLONG, float *, BLASLONG); +} openblas_sbmin_dispatch_t; +extern const openblas_sbmin_dispatch_t *const openblas_sbmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbamax_k)(BLASLONG, float *, BLASLONG); +} openblas_isbamax_dispatch_t; +extern const openblas_isbamax_dispatch_t *const openblas_isbamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbamin_k)(BLASLONG, float *, BLASLONG); +} openblas_isbamin_dispatch_t; +extern const openblas_isbamin_dispatch_t *const openblas_isbamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbmax_k) (BLASLONG, float *, BLASLONG); +} openblas_isbmax_dispatch_t; +extern const openblas_isbmax_dispatch_t *const openblas_isbmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { BLASLONG (*isbmin_k) (BLASLONG, float *, BLASLONG); +} openblas_isbmin_dispatch_t; +extern const openblas_isbmin_dispatch_t *const openblas_isbmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbnrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_sbnrm2_dispatch_t; +extern const openblas_sbnrm2_dispatch_t *const openblas_sbnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbasum_k) (BLASLONG, float *, BLASLONG); +} openblas_sbasum_dispatch_t; +extern const openblas_sbasum_dispatch_t *const openblas_sbasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbsum_k) (BLASLONG, float *, BLASLONG); +} openblas_sbsum_dispatch_t; +extern const openblas_sbsum_dispatch_t *const openblas_sbsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbcopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbcopy_dispatch_t; +extern const openblas_sbcopy_dispatch_t *const openblas_sbcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { float (*sbdot_k) (BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_sbdot_dispatch_t; +extern const openblas_sbdot_dispatch_t *const openblas_sbdot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { double (*dsbdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_dsbdot_dispatch_t; +extern const openblas_dsbdot_dispatch_t *const openblas_dsbdot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_sbrot_dispatch_t; +extern const openblas_sbrot_dispatch_t *const openblas_sbrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbrotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbrotm_dispatch_t; +extern const openblas_sbrotm_dispatch_t *const openblas_sbrotm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bscal_k) (BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); +} openblas_bscal_dispatch_t; +extern const openblas_bscal_dispatch_t *const openblas_bscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbaxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbaxpy_dispatch_t; +extern const openblas_sbaxpy_dispatch_t *const openblas_sbaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbscal_dispatch_t; +extern const openblas_sbscal_dispatch_t *const openblas_sbscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sbswap_dispatch_t; +extern const openblas_sbswap_dispatch_t *const openblas_sbswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bgemv_n) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); int (*bgemv_t) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +} openblas_bgemv_dispatch_t; +extern const openblas_bgemv_dispatch_t *const openblas_bgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbgemv_n) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbgemv_t) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); +} openblas_sbgemv_dispatch_t; +extern const openblas_sbgemv_dispatch_t *const openblas_sbgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbger_dispatch_t; +extern const openblas_sbger_dispatch_t *const openblas_sbger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbsymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sbsymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sbsymv_dispatch_t; +extern const openblas_sbsymv_dispatch_t *const openblas_sbsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*bgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); int (*bgemm_beta )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); - int (*bgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*bgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); +} openblas_bgemm_dispatch_t; +extern const openblas_bgemm_dispatch_t *const openblas_bgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int (*sbgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); - int (*sbgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); int (*sbgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); +#ifdef SMALL_MATRIX_OPT + int (*sbgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); + int (*sbgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); + int (*sbgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); +#endif +} openblas_sbgemm_dispatch_t; +extern const openblas_sbgemm_dispatch_t *const openblas_sbgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*sbtrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); @@ -166,12 +389,16 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sbtrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); int (*sbtrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_sbtrsm_dispatch_t; +extern const openblas_sbtrsm_dispatch_t *const openblas_sbtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*sbtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*sbtrmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -188,84 +415,185 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*sbtrmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbtrmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_sbtrmm_dispatch_t; +extern const openblas_sbtrmm_dispatch_t *const openblas_sbtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbsymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*sbsymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_sbsymm_dispatch_t; +extern const openblas_sbsymm_dispatch_t *const openblas_sbsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if BUILD_BFLOAT16 == 1 +typedef struct { int (*sbneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*sblaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); - -#ifdef SMALL_MATRIX_OPT - int (*sbgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); - - int (*sbgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - - int (*sbgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*sbgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc); -#endif +} openblas_sbneg_dispatch_t; +extern const openblas_sbneg_dispatch_t *const openblas_sbneg_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_SINGLE == 1) || (BUILD_DOUBLE == 1) || (BUILD_COMPLEX == 1) || (BUILD_COMPLEX16 == 1) - int sgemm_p, sgemm_q, sgemm_r; - int sgemm_unroll_m, sgemm_unroll_n, sgemm_unroll_mn; +#if BUILD_BFLOAT16 == 1 +typedef struct { + int (*sblaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_sblaswp_dispatch_t; +extern const openblas_sblaswp_dispatch_t *const openblas_sblaswp_dispatch[OPENBLAS_NUM_CORES]; #endif - int exclusive_cache; - #if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*samax_k) (BLASLONG, float *, BLASLONG); +} openblas_samax_dispatch_t; +extern const openblas_samax_dispatch_t *const openblas_samax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*samin_k) (BLASLONG, float *, BLASLONG); +} openblas_samin_dispatch_t; +extern const openblas_samin_dispatch_t *const openblas_samin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*smax_k) (BLASLONG, float *, BLASLONG); +} openblas_smax_dispatch_t; +extern const openblas_smax_dispatch_t *const openblas_smax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +typedef struct { float (*smin_k) (BLASLONG, float *, BLASLONG); +} openblas_smin_dispatch_t; +extern const openblas_smin_dispatch_t *const openblas_smin_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*isamax_k)(BLASLONG, float *, BLASLONG); +} openblas_isamax_dispatch_t; +extern const openblas_isamax_dispatch_t *const openblas_isamax_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*isamin_k)(BLASLONG, float *, BLASLONG); +} openblas_isamin_dispatch_t; +extern const openblas_isamin_dispatch_t *const openblas_isamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*ismax_k) (BLASLONG, float *, BLASLONG); +} openblas_ismax_dispatch_t; +extern const openblas_ismax_dispatch_t *const openblas_ismax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*ismin_k) (BLASLONG, float *, BLASLONG); +} openblas_ismin_dispatch_t; +extern const openblas_ismin_dispatch_t *const openblas_ismin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*snrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_snrm2_dispatch_t; +extern const openblas_snrm2_dispatch_t *const openblas_snrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*sasum_k) (BLASLONG, float *, BLASLONG); +} openblas_sasum_dispatch_t; +extern const openblas_sasum_dispatch_t *const openblas_sasum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) +typedef struct { float (*ssum_k) (BLASLONG, float *, BLASLONG); -#endif +} openblas_ssum_dispatch_t; +extern const openblas_ssum_dispatch_t *const openblas_ssum_dispatch[OPENBLAS_NUM_CORES]; +#endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*scopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_scopy_dispatch_t; +extern const openblas_scopy_dispatch_t *const openblas_scopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { float (*sdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); - //double (*dsdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sdot_dispatch_t; +extern const openblas_sdot_dispatch_t *const openblas_sdot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*srot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_srot_dispatch_t; +extern const openblas_srot_dispatch_t *const openblas_srot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*srotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_srotm_dispatch_t; +extern const openblas_srotm_dispatch_t *const openblas_srotm_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*saxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_saxpy_dispatch_t; +extern const openblas_saxpy_dispatch_t *const openblas_saxpy_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*sscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sscal_dispatch_t; +extern const openblas_sscal_dispatch_t *const openblas_sscal_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*sswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_sswap_dispatch_t; +extern const openblas_sswap_dispatch_t *const openblas_sswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { int (*sgemv_n) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sgemv_t) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sgemv_dispatch_t; +extern const openblas_sgemv_dispatch_t *const openblas_sgemv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) +typedef struct { int (*sger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_sger_dispatch_t; +extern const openblas_sger_dispatch_t *const openblas_sger_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*ssymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*ssymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_ssymv_dispatch_t; +extern const openblas_ssymv_dispatch_t *const openblas_ssymv_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { #ifdef ARCH_X86_64 void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); int (*sgemm_direct_performant) (BLASLONG M, BLASLONG N, BLASLONG K); @@ -274,77 +602,62 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); int (*sgemm_direct_performant) (BLASLONG M, BLASLONG N, BLASLONG K); void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*ssymm_direct_alpha_betaLU) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*ssymm_direct_alpha_betaLL) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); - void (*strmm_direct_LNUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LNLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LTUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*strmm_direct_LTLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyrk_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); - void (*ssyr2k_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); void (*sme_sgemm_kernel) (char*, char*, BLASLONG, BLASLONG, BLASLONG, float*, float *, BLASLONG , float *, BLASLONG ,float*, float *, BLASLONG); #endif - - int (*sgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int (*sgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - int (*ssymm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); - int (*strmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); - int (*sgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_oncopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*sgemm_otcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); -#endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) #ifdef SMALL_MATRIX_OPT int (*sgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); - int (*sgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); int (*sgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); - int (*sgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*sgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc); #endif +} openblas_sgemm_dispatch_t; +extern const openblas_sgemm_dispatch_t *const openblas_sgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*strsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { +#ifdef ARCH_ARM64 + void (*ssymm_direct_alpha_betaLU) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); + void (*ssymm_direct_alpha_betaLL) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); +#endif + int (*ssymm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); +#if (BUILD_SINGLE==1) + int (*ssymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*ssymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*ssymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#endif +} openblas_ssymm_dispatch_t; +extern const openblas_ssymm_dispatch_t *const openblas_ssymm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*strsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*strsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { +#ifdef ARCH_ARM64 + void (*strmm_direct_LNUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LNLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LTUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); + void (*strmm_direct_LTLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG); #endif + int (*strmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); #if (BUILD_SINGLE==1) int (*strmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); int (*strmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*strmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -353,8 +666,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*ssymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*strmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -365,116 +676,280 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*strmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*strmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#endif +} openblas_strmm_dispatch_t; +extern const openblas_strmm_dispatch_t *const openblas_strmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ssymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*ssymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +typedef struct { + void (*ssyrk_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyrk_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG); +} openblas_ssyrk_dispatch_t; +extern const openblas_ssyrk_dispatch_t *const openblas_ssyrk_dispatch[OPENBLAS_NUM_CORES]; +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +typedef struct { + void (*ssyr2k_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); + void (*ssyr2k_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float *, BLASLONG); +} openblas_ssyr2k_dispatch_t; +extern const openblas_ssyr2k_dispatch_t *const openblas_ssyr2k_dispatch[OPENBLAS_NUM_CORES]; +#endif +#endif +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +typedef struct { + int (*strsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*strsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*strsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_strsm_dispatch_t; +extern const openblas_strsm_dispatch_t *const openblas_strsm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*sneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); +} openblas_sneg_dispatch_t; +extern const openblas_sneg_dispatch_t *const openblas_sneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_SINGLE==1) +typedef struct { int (*slaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_slaswp_dispatch_t; +extern const openblas_slaswp_dispatch_t *const openblas_slaswp_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - int dgemm_p, dgemm_q, dgemm_r; - int dgemm_unroll_m, dgemm_unroll_n, dgemm_unroll_mn; +typedef struct { + double (*damax_k) (BLASLONG, double *, BLASLONG); +} openblas_damax_dispatch_t; +extern const openblas_damax_dispatch_t *const openblas_damax_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - double (*damax_k) (BLASLONG, double *, BLASLONG); +typedef struct { double (*damin_k) (BLASLONG, double *, BLASLONG); +} openblas_damin_dispatch_t; +extern const openblas_damin_dispatch_t *const openblas_damin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dmax_k) (BLASLONG, double *, BLASLONG); +} openblas_dmax_dispatch_t; +extern const openblas_dmax_dispatch_t *const openblas_dmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dmin_k) (BLASLONG, double *, BLASLONG); +} openblas_dmin_dispatch_t; +extern const openblas_dmin_dispatch_t *const openblas_dmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idamax_k)(BLASLONG, double *, BLASLONG); +} openblas_idamax_dispatch_t; +extern const openblas_idamax_dispatch_t *const openblas_idamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idamin_k)(BLASLONG, double *, BLASLONG); +} openblas_idamin_dispatch_t; +extern const openblas_idamin_dispatch_t *const openblas_idamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idmax_k) (BLASLONG, double *, BLASLONG); +} openblas_idmax_dispatch_t; +extern const openblas_idmax_dispatch_t *const openblas_idmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { BLASLONG (*idmin_k) (BLASLONG, double *, BLASLONG); +} openblas_idmin_dispatch_t; +extern const openblas_idmin_dispatch_t *const openblas_idmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dnrm2_k) (BLASLONG, double *, BLASLONG); +} openblas_dnrm2_dispatch_t; +extern const openblas_dnrm2_dispatch_t *const openblas_dnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*dasum_k) (BLASLONG, double *, BLASLONG); +} openblas_dasum_dispatch_t; +extern const openblas_dasum_dispatch_t *const openblas_dasum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { double (*dsum_k) (BLASLONG, double *, BLASLONG); +} openblas_dsum_dispatch_t; +extern const openblas_dsum_dispatch_t *const openblas_dsum_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dcopy_dispatch_t; +extern const openblas_dcopy_dispatch_t *const openblas_dcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { double (*ddot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_ddot_dispatch_t; +extern const openblas_ddot_dispatch_t *const openblas_ddot_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +typedef struct { double (*dsdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_dsdot_dispatch_t; +extern const openblas_dsdot_dispatch_t *const openblas_dsdot_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*drot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double, double); +} openblas_drot_dispatch_t; +extern const openblas_drot_dispatch_t *const openblas_drot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*drotm_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_drotm_dispatch_t; +extern const openblas_drotm_dispatch_t *const openblas_drotm_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*daxpy_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_daxpy_dispatch_t; +extern const openblas_daxpy_dispatch_t *const openblas_daxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dscal_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dscal_dispatch_t; +extern const openblas_dscal_dispatch_t *const openblas_dscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dswap_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_dswap_dispatch_t; +extern const openblas_dswap_dispatch_t *const openblas_dswap_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { int (*dgemv_n) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*dgemv_t) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dgemv_dispatch_t; +extern const openblas_dgemv_dispatch_t *const openblas_dgemv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*dger_k) (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dger_dispatch_t; +extern const openblas_dger_dispatch_t *const openblas_dger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) +typedef struct { int (*dsymv_L) (BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*dsymv_U) (BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_dsymv_dispatch_t; +extern const openblas_dsymv_dispatch_t *const openblas_dsymv_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_dgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const double*, const double *, const BLASLONG , const double *, const BLASLONG ,const double*, double *, const BLASLONG); #endif int (*dgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); int (*dgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - int (*dsymm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); - int (*dtrmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); - int (*dgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_oncopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dgemm_otcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); -#endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) #ifdef SMALL_MATRIX_OPT int (*dgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double beta); - int (*dgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); int (*dgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); - int (*dgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*dgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); #endif +} openblas_dgemm_dispatch_t; +extern const openblas_dgemm_dispatch_t *const openblas_dgemm_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_DOUBLE==1) - int (*dtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*dtrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { + int (*dsymm_kernel )(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +#if (BUILD_DOUBLE==1) + int (*dsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*dsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*dsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#endif +} openblas_dsymm_dispatch_t; +extern const openblas_dsymm_dispatch_t *const openblas_dsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +typedef struct { + int (*dtrmm_gemm_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); +#if (BUILD_DOUBLE==1) int (*dtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); int (*dtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*dtrmm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -483,8 +958,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*dsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*dtrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -495,61 +968,218 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*dtrmm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*dtrmm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#endif +} openblas_dtrmm_dispatch_t; +extern const openblas_dtrmm_dispatch_t *const openblas_dtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*dsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*dsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#if (BUILD_DOUBLE==1) +typedef struct { + int (*dtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*dtrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*dtrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +} openblas_dtrsm_dispatch_t; +extern const openblas_dtrsm_dispatch_t *const openblas_dtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_DOUBLE==1) +typedef struct { int (*dneg_tcopy) (BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*dlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_dneg_dispatch_t; +extern const openblas_dneg_dispatch_t *const openblas_dneg_dispatch[OPENBLAS_NUM_CORES]; #endif -#ifdef EXPRECISION - int qgemm_p, qgemm_q, qgemm_r; - int qgemm_unroll_m, qgemm_unroll_n, qgemm_unroll_mn; +#if (BUILD_DOUBLE==1) +typedef struct { + int (*dlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_dlaswp_dispatch_t; +extern const openblas_dlaswp_dispatch_t *const openblas_dlaswp_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*qamax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qamax_dispatch_t; +extern const openblas_qamax_dispatch_t *const openblas_qamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qamin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qamin_dispatch_t; +extern const openblas_qamin_dispatch_t *const openblas_qamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qmax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qmax_dispatch_t; +extern const openblas_qmax_dispatch_t *const openblas_qmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qmin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qmin_dispatch_t; +extern const openblas_qmin_dispatch_t *const openblas_qmin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqamax_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_iqamax_dispatch_t; +extern const openblas_iqamax_dispatch_t *const openblas_iqamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqamin_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_iqamin_dispatch_t; +extern const openblas_iqamin_dispatch_t *const openblas_iqamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqmax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_iqmax_dispatch_t; +extern const openblas_iqmax_dispatch_t *const openblas_iqmax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*iqmin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_iqmin_dispatch_t; +extern const openblas_iqmin_dispatch_t *const openblas_iqmin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*qnrm2_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qnrm2_dispatch_t; +extern const openblas_qnrm2_dispatch_t *const openblas_qnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qasum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qasum_dispatch_t; +extern const openblas_qasum_dispatch_t *const openblas_qasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qsum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_qsum_dispatch_t; +extern const openblas_qsum_dispatch_t *const openblas_qsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qcopy_dispatch_t; +extern const openblas_qcopy_dispatch_t *const openblas_qcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*qdot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qdot_dispatch_t; +extern const openblas_qdot_dispatch_t *const openblas_qdot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qrot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble); +} openblas_qrot_dispatch_t; +extern const openblas_qrot_dispatch_t *const openblas_qrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qrotm_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qrotm_dispatch_t; +extern const openblas_qrotm_dispatch_t *const openblas_qrotm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qaxpy_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qaxpy_dispatch_t; +extern const openblas_qaxpy_dispatch_t *const openblas_qaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qscal_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qscal_dispatch_t; +extern const openblas_qscal_dispatch_t *const openblas_qscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qswap_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_qswap_dispatch_t; +extern const openblas_qswap_dispatch_t *const openblas_qswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qgemv_n) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemv_t) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qgemv_dispatch_t; +extern const openblas_qgemv_dispatch_t *const openblas_qgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*qger_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qger_dispatch_t; +extern const openblas_qger_dispatch_t *const openblas_qger_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qsymv_L) (BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qsymv_U) (BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qsymv_dispatch_t; +extern const openblas_qsymv_dispatch_t *const openblas_qsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*qgemm_beta )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); - int (*qgemm_incopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_itcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_oncopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*qgemm_otcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_qgemm_dispatch_t; +extern const openblas_qgemm_dispatch_t *const openblas_qgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*qtrsm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); @@ -566,12 +1196,16 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*qtrsm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*qtrsm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); +} openblas_qtrsm_dispatch_t; +extern const openblas_qtrsm_dispatch_t *const openblas_qtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*qtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*qtrmm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); @@ -588,41 +1222,141 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*qtrmm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qtrmm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_qtrmm_dispatch_t; +extern const openblas_qtrmm_dispatch_t *const openblas_qtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qsymm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*qsymm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_qsymm_dispatch_t; +extern const openblas_qsymm_dispatch_t *const openblas_qsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*qneg_tcopy) (BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*qlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_qneg_dispatch_t; +extern const openblas_qneg_dispatch_t *const openblas_qneg_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { + int (*qlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_qlaswp_dispatch_t; +extern const openblas_qlaswp_dispatch_t *const openblas_qlaswp_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_COMPLEX==1) - int cgemm_p, cgemm_q, cgemm_r; - int cgemm_unroll_m, cgemm_unroll_n, cgemm_unroll_mn; - +#if (BUILD_COMPLEX==1) +typedef struct { float (*camax_k) (BLASLONG, float *, BLASLONG); +} openblas_camax_dispatch_t; +extern const openblas_camax_dispatch_t *const openblas_camax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*camin_k) (BLASLONG, float *, BLASLONG); - +} openblas_camin_dispatch_t; +extern const openblas_camin_dispatch_t *const openblas_camin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*icamax_k)(BLASLONG, float *, BLASLONG); +} openblas_icamax_dispatch_t; +extern const openblas_icamax_dispatch_t *const openblas_icamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { BLASLONG (*icamin_k)(BLASLONG, float *, BLASLONG); +} openblas_icamin_dispatch_t; +extern const openblas_icamin_dispatch_t *const openblas_icamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { float (*cnrm2_k) (BLASLONG, float *, BLASLONG); +} openblas_cnrm2_dispatch_t; +extern const openblas_cnrm2_dispatch_t *const openblas_cnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*casum_k) (BLASLONG, float *, BLASLONG); +} openblas_casum_dispatch_t; +extern const openblas_casum_dispatch_t *const openblas_casum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { float (*csum_k) (BLASLONG, float *, BLASLONG); +} openblas_csum_dispatch_t; +extern const openblas_csum_dispatch_t *const openblas_csum_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*ccopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_ccopy_dispatch_t; +extern const openblas_ccopy_dispatch_t *const openblas_ccopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { openblas_complex_float (*cdotu_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cdotu_dispatch_t; +extern const openblas_cdotu_dispatch_t *const openblas_cdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { openblas_complex_float (*cdotc_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cdotc_dispatch_t; +extern const openblas_cdotc_dispatch_t *const openblas_cdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); +} openblas_csrot_dispatch_t; +extern const openblas_csrot_dispatch_t *const openblas_csrot_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*caxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_caxpy_dispatch_t; +extern const openblas_caxpy_dispatch_t *const openblas_caxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*caxpyc_k)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_caxpyc_dispatch_t; +extern const openblas_caxpyc_dispatch_t *const openblas_caxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cscal_dispatch_t; +extern const openblas_cscal_dispatch_t *const openblas_cscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); +} openblas_cswap_dispatch_t; +extern const openblas_cswap_dispatch_t *const openblas_cswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgemv_n) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_t) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_r) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); @@ -631,18 +1365,58 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemv_u) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_s) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*cgemv_d) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgemv_dispatch_t; +extern const openblas_cgemv_dispatch_t *const openblas_cgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgeru_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgeru_dispatch_t; +extern const openblas_cgeru_dispatch_t *const openblas_cgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerc_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerc_dispatch_t; +extern const openblas_cgerc_dispatch_t *const openblas_cgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerv_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerv_dispatch_t; +extern const openblas_cgerv_dispatch_t *const openblas_cgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgerd_k) (BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_cgerd_dispatch_t; +extern const openblas_cgerd_dispatch_t *const openblas_cgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csymv_L) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*csymv_U) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_csymv_dispatch_t; +extern const openblas_csymv_dispatch_t *const openblas_csymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemv_L) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_U) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_M) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*chemv_V) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); +} openblas_chemv_dispatch_t; +extern const openblas_chemv_dispatch_t *const openblas_chemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_cgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const float, const float, const float *, const BLASLONG , const float *, const BLASLONG, const float, const float, float *, const BLASLONG); #endif @@ -651,92 +1425,71 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*cgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*cgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); - - int (*csymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*csymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - - int (*ctrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*ctrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*cgemm_incopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_itcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_oncopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm_otcopy )(BLASLONG, BLASLONG, float *, BLASLONG, float *); - #ifdef SMALL_MATRIX_OPT int (*cgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha0, float alpha1, float beta0, float beta1); - int (*cgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_nc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_tc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_rn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_rc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_cn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_ct )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_cr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); int (*cgemm_small_kernel_cc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); - int (*cgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); int (*cgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float * C, BLASLONG ldc); #endif +} openblas_cgemm_dispatch_t; +extern const openblas_cgemm_dispatch_t *const openblas_cgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ctrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - - int (*ctrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); - int (*ctrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +#if (BUILD_COMPLEX==1) +typedef struct { + int (*csymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*csymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); + int (*csymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); + int (*csymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_csymm_dispatch_t; +extern const openblas_csymm_dispatch_t *const openblas_csymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { + int (*ctrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); + int (*ctrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); int (*ctrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); @@ -745,7 +1498,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); int (*ctrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); - int (*ctrmm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -754,8 +1506,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*csymm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_incopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_itcopy)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*ctrmm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); @@ -766,90 +1516,226 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ctrmm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*ctrmm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_ctrmm_dispatch_t; +extern const openblas_ctrmm_dispatch_t *const openblas_ctrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*csymm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +#if (BUILD_COMPLEX==1) +typedef struct { + int (*ctrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG); + int (*ctrsm_iunucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iunncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iutucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iutncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ilnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ilnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_iltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ounucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_ounncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_outucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_outncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_olnucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_olnncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_oltucopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); + int (*ctrsm_oltncopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, float *); +} openblas_ctrsm_dispatch_t; +extern const openblas_ctrsm_dispatch_t *const openblas_ctrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemm_iutcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_iltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_outcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm_oltcopy)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); +} openblas_chemm_dispatch_t; +extern const openblas_chemm_dispatch_t *const openblas_chemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int cgemm3m_p, cgemm3m_q, cgemm3m_r; - int cgemm3m_unroll_m, cgemm3m_unroll_n, cgemm3m_unroll_mn; - +#if (BUILD_COMPLEX==1) +typedef struct { int (*cgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG); - int (*cgemm3m_incopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_incopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_incopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float *); int (*cgemm3m_itcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float *); - int (*cgemm3m_oncopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_oncopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_oncopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); int (*cgemm3m_otcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, float, float, float *); +} openblas_cgemm3m_dispatch_t; +extern const openblas_cgemm3m_dispatch_t *const openblas_cgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*csymm3m_iucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_iucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_iucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*csymm3m_ilcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*csymm3m_oucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_oucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_oucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*csymm3m_olcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); +} openblas_csymm3m_dispatch_t; +extern const openblas_csymm3m_dispatch_t *const openblas_csymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*chemm3m_iucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_iucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_iucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); int (*chemm3m_ilcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float *); - int (*chemm3m_oucopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyb)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_oucopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyr)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_oucopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); int (*chemm3m_olcopyi)(BLASLONG, BLASLONG, float *, BLASLONG, BLASLONG, BLASLONG, float, float, float *); +} openblas_chemm3m_dispatch_t; +extern const openblas_chemm3m_dispatch_t *const openblas_chemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX==1) +typedef struct { int (*cneg_tcopy) (BLASLONG, BLASLONG, float *, BLASLONG, float *); +} openblas_cneg_dispatch_t; +extern const openblas_cneg_dispatch_t *const openblas_cneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX==1) +typedef struct { int (*claswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *); +} openblas_claswp_dispatch_t; +extern const openblas_claswp_dispatch_t *const openblas_claswp_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16 == 1) - int zgemm_p, zgemm_q, zgemm_r; - int zgemm_unroll_m, zgemm_unroll_n, zgemm_unroll_mn; - +typedef struct { double (*zamax_k) (BLASLONG, double *, BLASLONG); +} openblas_zamax_dispatch_t; +extern const openblas_zamax_dispatch_t *const openblas_zamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zamin_k) (BLASLONG, double *, BLASLONG); +} openblas_zamin_dispatch_t; +extern const openblas_zamin_dispatch_t *const openblas_zamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { BLASLONG (*izamax_k)(BLASLONG, double *, BLASLONG); +} openblas_izamax_dispatch_t; +extern const openblas_izamax_dispatch_t *const openblas_izamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { BLASLONG (*izamin_k)(BLASLONG, double *, BLASLONG); +} openblas_izamin_dispatch_t; +extern const openblas_izamin_dispatch_t *const openblas_izamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*znrm2_k) (BLASLONG, double *, BLASLONG); +} openblas_znrm2_dispatch_t; +extern const openblas_znrm2_dispatch_t *const openblas_znrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zasum_k) (BLASLONG, double *, BLASLONG); +} openblas_zasum_dispatch_t; +extern const openblas_zasum_dispatch_t *const openblas_zasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { double (*zsum_k) (BLASLONG, double *, BLASLONG); +} openblas_zsum_dispatch_t; +extern const openblas_zsum_dispatch_t *const openblas_zsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zcopy_dispatch_t; +extern const openblas_zcopy_dispatch_t *const openblas_zcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { openblas_complex_double (*zdotu_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zdotu_dispatch_t; +extern const openblas_zdotu_dispatch_t *const openblas_zdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { openblas_complex_double (*zdotc_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zdotc_dispatch_t; +extern const openblas_zdotc_dispatch_t *const openblas_zdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zdrot_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG, double, double); +} openblas_zdrot_dispatch_t; +extern const openblas_zdrot_dispatch_t *const openblas_zdrot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zaxpy_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zaxpy_dispatch_t; +extern const openblas_zaxpy_dispatch_t *const openblas_zaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zaxpyc_k)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zaxpyc_dispatch_t; +extern const openblas_zaxpyc_dispatch_t *const openblas_zaxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zscal_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zscal_dispatch_t; +extern const openblas_zscal_dispatch_t *const openblas_zscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zswap_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); +} openblas_zswap_dispatch_t; +extern const openblas_zswap_dispatch_t *const openblas_zswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgemv_n) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_t) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_r) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); @@ -858,113 +1744,131 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*zgemv_u) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_s) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zgemv_d) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgemv_dispatch_t; +extern const openblas_zgemv_dispatch_t *const openblas_zgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgeru_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgeru_dispatch_t; +extern const openblas_zgeru_dispatch_t *const openblas_zgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerc_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerc_dispatch_t; +extern const openblas_zgerc_dispatch_t *const openblas_zgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerv_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerv_dispatch_t; +extern const openblas_zgerv_dispatch_t *const openblas_zgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgerd_k) (BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zgerd_dispatch_t; +extern const openblas_zgerd_dispatch_t *const openblas_zgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zsymv_L) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zsymv_U) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zsymv_dispatch_t; +extern const openblas_zsymv_dispatch_t *const openblas_zsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemv_L) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_U) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_M) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); int (*zhemv_V) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *); +} openblas_zhemv_dispatch_t; +extern const openblas_zhemv_dispatch_t *const openblas_zhemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { #ifdef ARCH_ARM64 void (*sme_zgemm_kernel) (const char*, const char*, const BLASLONG, const BLASLONG, const BLASLONG, const double, const double, const double *, const BLASLONG , const double *, const BLASLONG, const double, const double, double *, const BLASLONG); #endif - int (*zgemm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*zgemm_beta )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG); - - int (*zsymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zsymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - - int (*ztrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*ztrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zgemm_incopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_itcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_oncopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm_otcopy )(BLASLONG, BLASLONG, double *, BLASLONG, double *); - #ifdef SMALL_MATRIX_OPT int (*zgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha0, double alpha1, double beta0, double beta1); - int (*zgemm_small_kernel_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_nc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_tc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_rn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_rc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_cn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_ct )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_cr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); int (*zgemm_small_kernel_cc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); - int (*zgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); int (*zgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double * C, BLASLONG ldc); #endif +} openblas_zgemm_dispatch_t; +extern const openblas_zgemm_dispatch_t *const openblas_zgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*ztrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - - int (*ztrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); - int (*ztrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*zsymm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*zsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); + int (*zsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); + int (*zsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_zsymm_dispatch_t; +extern const openblas_zsymm_dispatch_t *const openblas_zsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*ztrmm_gemm_kernel_n)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_l)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_r)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); + int (*ztrmm_gemm_kernel_b)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); int (*ztrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); @@ -973,7 +1877,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); int (*ztrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); - int (*ztrmm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -982,8 +1885,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*zsymm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_incopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_itcopy)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*ztrmm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); @@ -994,91 +1895,226 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*ztrmm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*ztrmm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_ztrmm_dispatch_t; +extern const openblas_ztrmm_dispatch_t *const openblas_ztrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int (*zsymm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +#if (BUILD_COMPLEX16 == 1) +typedef struct { + int (*ztrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG); + int (*ztrsm_iunucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iunncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iutucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iutncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ilnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ilnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_iltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ounucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_ounncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_outucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_outncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_olnucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_olnncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_oltucopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); + int (*ztrsm_oltncopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, double *); +} openblas_ztrsm_dispatch_t; +extern const openblas_ztrsm_dispatch_t *const openblas_ztrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemm_iutcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_iltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_outcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm_oltcopy)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); +} openblas_zhemm_dispatch_t; +extern const openblas_zhemm_dispatch_t *const openblas_zhemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int zgemm3m_p, zgemm3m_q, zgemm3m_r; - int zgemm3m_unroll_m, zgemm3m_unroll_n, zgemm3m_unroll_mn; - +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG); - int (*zgemm3m_incopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_incopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_incopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double *); int (*zgemm3m_itcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double *); - int (*zgemm3m_oncopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_oncopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_oncopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); int (*zgemm3m_otcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, double, double, double *); +} openblas_zgemm3m_dispatch_t; +extern const openblas_zgemm3m_dispatch_t *const openblas_zgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zsymm3m_iucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_iucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_iucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zsymm3m_ilcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zsymm3m_oucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_oucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_oucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zsymm3m_olcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); +} openblas_zsymm3m_dispatch_t; +extern const openblas_zsymm3m_dispatch_t *const openblas_zsymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zhemm3m_iucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_iucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_iucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); int (*zhemm3m_ilcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double *); - int (*zhemm3m_oucopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyb)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_oucopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyr)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_oucopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); int (*zhemm3m_olcopyi)(BLASLONG, BLASLONG, double *, BLASLONG, BLASLONG, BLASLONG, double, double, double *); +} openblas_zhemm3m_dispatch_t; +extern const openblas_zhemm3m_dispatch_t *const openblas_zhemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zneg_tcopy) (BLASLONG, BLASLONG, double *, BLASLONG, double *); +} openblas_zneg_dispatch_t; +extern const openblas_zneg_dispatch_t *const openblas_zneg_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#if (BUILD_COMPLEX16 == 1) +typedef struct { int (*zlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, double *, BLASLONG, blasint *, double *); +} openblas_zlaswp_dispatch_t; +extern const openblas_zlaswp_dispatch_t *const openblas_zlaswp_dispatch[OPENBLAS_NUM_CORES]; #endif #ifdef EXPRECISION - - int xgemm_p, xgemm_q, xgemm_r; - int xgemm_unroll_m, xgemm_unroll_n, xgemm_unroll_mn; - +typedef struct { xdouble (*xamax_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xamax_dispatch_t; +extern const openblas_xamax_dispatch_t *const openblas_xamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xamin_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xamin_dispatch_t; +extern const openblas_xamin_dispatch_t *const openblas_xamin_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*ixamax_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_ixamax_dispatch_t; +extern const openblas_ixamax_dispatch_t *const openblas_ixamax_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { BLASLONG (*ixamin_k)(BLASLONG, xdouble *, BLASLONG); +} openblas_ixamin_dispatch_t; +extern const openblas_ixamin_dispatch_t *const openblas_ixamin_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { xdouble (*xnrm2_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xnrm2_dispatch_t; +extern const openblas_xnrm2_dispatch_t *const openblas_xnrm2_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xasum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xasum_dispatch_t; +extern const openblas_xasum_dispatch_t *const openblas_xasum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { xdouble (*xsum_k) (BLASLONG, xdouble *, BLASLONG); +} openblas_xsum_dispatch_t; +extern const openblas_xsum_dispatch_t *const openblas_xsum_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xcopy_dispatch_t; +extern const openblas_xcopy_dispatch_t *const openblas_xcopy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { openblas_complex_xdouble (*xdotu_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xdotu_dispatch_t; +extern const openblas_xdotu_dispatch_t *const openblas_xdotu_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { openblas_complex_xdouble (*xdotc_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xdotc_dispatch_t; +extern const openblas_xdotc_dispatch_t *const openblas_xdotc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xqrot_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble); +} openblas_xqrot_dispatch_t; +extern const openblas_xqrot_dispatch_t *const openblas_xqrot_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xaxpy_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xaxpy_dispatch_t; +extern const openblas_xaxpy_dispatch_t *const openblas_xaxpy_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xaxpyc_k)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xaxpyc_dispatch_t; +extern const openblas_xaxpyc_dispatch_t *const openblas_xaxpyc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xscal_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xscal_dispatch_t; +extern const openblas_xscal_dispatch_t *const openblas_xscal_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xswap_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); +} openblas_xswap_dispatch_t; +extern const openblas_xswap_dispatch_t *const openblas_xswap_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xgemv_n) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_t) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_r) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); @@ -1087,29 +2123,73 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xgemv_u) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_s) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemv_d) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgemv_dispatch_t; +extern const openblas_xgemv_dispatch_t *const openblas_xgemv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgeru_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgeru_dispatch_t; +extern const openblas_xgeru_dispatch_t *const openblas_xgeru_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerc_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerc_dispatch_t; +extern const openblas_xgerc_dispatch_t *const openblas_xgerc_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerv_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerv_dispatch_t; +extern const openblas_xgerv_dispatch_t *const openblas_xgerv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xgerd_k) (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgerd_dispatch_t; +extern const openblas_xgerd_dispatch_t *const openblas_xgerd_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymv_L) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xsymv_U) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xsymv_dispatch_t; +extern const openblas_xsymv_dispatch_t *const openblas_xsymv_dispatch[OPENBLAS_NUM_CORES]; +#endif + +#ifdef EXPRECISION +typedef struct { int (*xhemv_L) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_U) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_M) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xhemv_V) (BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xhemv_dispatch_t; +extern const openblas_xhemv_dispatch_t *const openblas_xhemv_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xgemm_kernel_n )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_l )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_r )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_kernel_b )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); int (*xgemm_beta )(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG); - int (*xgemm_incopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_itcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_oncopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm_otcopy )(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); +} openblas_xgemm_dispatch_t; +extern const openblas_xgemm_dispatch_t *const openblas_xgemm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); @@ -1118,7 +2198,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrsm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrsm_kernel_RC)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*xtrsm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); @@ -1135,7 +2214,12 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrsm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); int (*xtrsm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, xdouble *); +} openblas_xtrsm_dispatch_t; +extern const openblas_xtrsm_dispatch_t *const openblas_xtrsm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xtrmm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_RT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_RR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); @@ -1144,7 +2228,6 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrmm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); int (*xtrmm_kernel_LC)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG, BLASLONG); - int (*xtrmm_iunucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_iunncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_iutucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); @@ -1161,177 +2244,250 @@ int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BL int (*xtrmm_olnncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_oltucopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xtrmm_oltncopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xtrmm_dispatch_t; +extern const openblas_xtrmm_dispatch_t *const openblas_xtrmm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xsymm_dispatch_t; +extern const openblas_xsymm_dispatch_t *const openblas_xsymm_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xhemm_iutcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_iltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_outcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm_oltcopy)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); +} openblas_xhemm_dispatch_t; +extern const openblas_xhemm_dispatch_t *const openblas_xhemm_dispatch[OPENBLAS_NUM_CORES]; +#endif - int xgemm3m_p, xgemm3m_q, xgemm3m_r; - int xgemm3m_unroll_m, xgemm3m_unroll_n, xgemm3m_unroll_mn; - +#ifdef EXPRECISION +typedef struct { int (*xgemm3m_kernel)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG); - int (*xgemm3m_incopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_incopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_incopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); int (*xgemm3m_itcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*xgemm3m_oncopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_oncopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_oncopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); int (*xgemm3m_otcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xgemm3m_dispatch_t; +extern const openblas_xgemm3m_dispatch_t *const openblas_xgemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xsymm3m_iucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_iucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_iucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xsymm3m_ilcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); - int (*xsymm3m_oucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_oucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_oucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xsymm3m_olcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xsymm3m_dispatch_t; +extern const openblas_xsymm3m_dispatch_t *const openblas_xsymm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xhemm3m_iucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_iucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_iucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); int (*xhemm3m_ilcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble *); - int (*xhemm3m_oucopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyb)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_oucopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyr)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_oucopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); int (*xhemm3m_olcopyi)(BLASLONG, BLASLONG, xdouble *, BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, xdouble *); +} openblas_xhemm3m_dispatch_t; +extern const openblas_xhemm3m_dispatch_t *const openblas_xhemm3m_dispatch[OPENBLAS_NUM_CORES]; +#endif +#ifdef EXPRECISION +typedef struct { int (*xneg_tcopy) (BLASLONG, BLASLONG, xdouble *, BLASLONG, xdouble *); - int (*xlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); - +} openblas_xneg_dispatch_t; +extern const openblas_xneg_dispatch_t *const openblas_xneg_dispatch[OPENBLAS_NUM_CORES]; #endif +#ifdef EXPRECISION +typedef struct { + int (*xlaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, xdouble *, BLASLONG, blasint *, xdouble *); +} openblas_xlaswp_dispatch_t; +extern const openblas_xlaswp_dispatch_t *const openblas_xlaswp_dispatch[OPENBLAS_NUM_CORES]; +#endif - void (*init)(void); - - int snum_opt, dnum_opt, qnum_opt; #if (BUILD_SINGLE==1) +typedef struct { int (*saxpby_k) (BLASLONG, float, float*, BLASLONG,float, float*, BLASLONG); +} openblas_saxpby_dispatch_t; +extern const openblas_saxpby_dispatch_t *const openblas_saxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*daxpby_k) (BLASLONG, double, double*, BLASLONG,double, double*, BLASLONG); +} openblas_daxpby_dispatch_t; +extern const openblas_daxpby_dispatch_t *const openblas_daxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX==1) +typedef struct { int (*caxpby_k) (BLASLONG, float, float, float*, BLASLONG,float,float, float*, BLASLONG); +} openblas_caxpby_dispatch_t; +extern const openblas_caxpby_dispatch_t *const openblas_caxpby_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX16==1) +typedef struct { int (*zaxpby_k) (BLASLONG, double, double, double*, BLASLONG,double,double, double*, BLASLONG); +} openblas_zaxpby_dispatch_t; +extern const openblas_zaxpby_dispatch_t *const openblas_zaxpby_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*somatcopy_k_cn) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_ct) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_rn) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); int (*somatcopy_k_rt) (BLASLONG, BLASLONG, float, float*, BLASLONG, float*, BLASLONG); +} openblas_somatcopy_dispatch_t; +extern const openblas_somatcopy_dispatch_t *const openblas_somatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif -#if (BUILD_DOUBLE==1) +#if (BUILD_DOUBLE==1) +typedef struct { int (*domatcopy_k_cn) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_ct) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_rn) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); int (*domatcopy_k_rt) (BLASLONG, BLASLONG, double, double*, BLASLONG, double*, BLASLONG); +} openblas_domatcopy_dispatch_t; +extern const openblas_domatcopy_dispatch_t *const openblas_domatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX==1) +typedef struct { int (*comatcopy_k_cn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_ct) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rt) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); - int (*comatcopy_k_cnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_ctc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); int (*comatcopy_k_rtc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG, float*, BLASLONG); +} openblas_comatcopy_dispatch_t; +extern const openblas_comatcopy_dispatch_t *const openblas_comatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16==1) +typedef struct { int (*zomatcopy_k_cn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_ct) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rt) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); - int (*zomatcopy_k_cnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_ctc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); int (*zomatcopy_k_rtc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG, double*, BLASLONG); +} openblas_zomatcopy_dispatch_t; +extern const openblas_zomatcopy_dispatch_t *const openblas_zomatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*simatcopy_k_cn) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_ct) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_rn) (BLASLONG, BLASLONG, float, float*, BLASLONG); int (*simatcopy_k_rt) (BLASLONG, BLASLONG, float, float*, BLASLONG); +} openblas_simatcopy_dispatch_t; +extern const openblas_simatcopy_dispatch_t *const openblas_simatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_DOUBLE==1) +typedef struct { int (*dimatcopy_k_cn) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_ct) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_rn) (BLASLONG, BLASLONG, double, double*, BLASLONG); int (*dimatcopy_k_rt) (BLASLONG, BLASLONG, double, double*, BLASLONG); +} openblas_dimatcopy_dispatch_t; +extern const openblas_dimatcopy_dispatch_t *const openblas_dimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX==1) +typedef struct { int (*cimatcopy_k_cn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_ct) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rn) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rt) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); - int (*cimatcopy_k_cnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_ctc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rnc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); int (*cimatcopy_k_rtc) (BLASLONG, BLASLONG, float, float, float*, BLASLONG); +} openblas_cimatcopy_dispatch_t; +extern const openblas_cimatcopy_dispatch_t *const openblas_cimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_COMPLEX16==1) +typedef struct { int (*zimatcopy_k_cn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_ct) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rn) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rt) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); - int (*zimatcopy_k_cnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_ctc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rnc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); int (*zimatcopy_k_rtc) (BLASLONG, BLASLONG, double, double, double*, BLASLONG); +} openblas_zimatcopy_dispatch_t; +extern const openblas_zimatcopy_dispatch_t *const openblas_zimatcopy_dispatch[OPENBLAS_NUM_CORES]; #endif #if (BUILD_SINGLE==1) +typedef struct { int (*sgeadd_k) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG, BLASLONG, BLASLONG); +} openblas_sgeadd_dispatch_t; +extern const openblas_sgeadd_dispatch_t *const openblas_sgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_DOUBLE==1) +typedef struct { int (*dgeadd_k) (BLASLONG, BLASLONG, double, double *, BLASLONG, double, double *, BLASLONG, BLASLONG, BLASLONG); +} openblas_dgeadd_dispatch_t; +extern const openblas_dgeadd_dispatch_t *const openblas_dgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX==1) +typedef struct { int (*cgeadd_k) (BLASLONG, BLASLONG, float, float, float *, BLASLONG, float, float, float *, BLASLONG, BLASLONG, BLASLONG); +} openblas_cgeadd_dispatch_t; +extern const openblas_cgeadd_dispatch_t *const openblas_cgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif + #if (BUILD_COMPLEX16==1) +typedef struct { int (*zgeadd_k) (BLASLONG, BLASLONG, double, double, double *, BLASLONG, double, double, double *, BLASLONG, BLASLONG, BLASLONG); +} openblas_zgeadd_dispatch_t; +extern const openblas_zgeadd_dispatch_t *const openblas_zgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif -} gotoblas_t; - -extern gotoblas_t *gotoblas; #define FUNC_OFFSET(func) (size_t)(&((gotoblas_t *)NULL)->func) /* The address that FUNC_OFFSET(func) is an offset from. */ diff --git a/common_q.h b/common_q.h index 52da71c88c..9607fdebfa 100644 --- a/common_q.h +++ b/common_q.h @@ -147,116 +147,116 @@ #else -#define QAMAX_K gotoblas -> qamax_k -#define QAMIN_K gotoblas -> qamin_k -#define QMAX_K gotoblas -> qmax_k -#define QMIN_K gotoblas -> qmin_k -#define IQAMAX_K gotoblas -> iqamax_k -#define IQAMIN_K gotoblas -> iqamin_k -#define IQMAX_K gotoblas -> iqmax_k -#define IQMIN_K gotoblas -> iqmin_k -#define QASUM_K gotoblas -> qasum_k -#define QAXPYU_K gotoblas -> qaxpy_k -#define QAXPYC_K gotoblas -> qaxpy_k -#define QCOPY_K gotoblas -> qcopy_k -#define QDOTU_K gotoblas -> qdot_k -#define QDOTC_K gotoblas -> qdot_k -#define QNRM2_K gotoblas -> qnrm2_k -#define QSCAL_K gotoblas -> qscal_k -#define QSUM_K gotoblas -> qsum_k -#define QSWAP_K gotoblas -> qswap_k -#define QROT_K gotoblas -> qrot_k -#define QROTM_K gotoblas -> qrotm_k - -#define QGEMV_N gotoblas -> qgemv_n -#define QGEMV_T gotoblas -> qgemv_t -#define QGEMV_R gotoblas -> qgemv_n -#define QGEMV_C gotoblas -> qgemv_t -#define QGEMV_O gotoblas -> qgemv_n -#define QGEMV_U gotoblas -> qgemv_t -#define QGEMV_S gotoblas -> qgemv_n -#define QGEMV_D gotoblas -> qgemv_t - -#define QGERU_K gotoblas -> qger_k -#define QGERC_K gotoblas -> qger_k -#define QGERV_K gotoblas -> qger_k -#define QGERD_K gotoblas -> qger_k - -#define QSYMV_U gotoblas -> qsymv_U -#define QSYMV_L gotoblas -> qsymv_L +#define QAMAX_K OPENBLAS_DISPATCH(qamax) -> qamax_k +#define QAMIN_K OPENBLAS_DISPATCH(qamin) -> qamin_k +#define QMAX_K OPENBLAS_DISPATCH(qmax) -> qmax_k +#define QMIN_K OPENBLAS_DISPATCH(qmin) -> qmin_k +#define IQAMAX_K OPENBLAS_DISPATCH(iqamax) -> iqamax_k +#define IQAMIN_K OPENBLAS_DISPATCH(iqamin) -> iqamin_k +#define IQMAX_K OPENBLAS_DISPATCH(iqmax) -> iqmax_k +#define IQMIN_K OPENBLAS_DISPATCH(iqmin) -> iqmin_k +#define QASUM_K OPENBLAS_DISPATCH(qasum) -> qasum_k +#define QAXPYU_K OPENBLAS_DISPATCH(qaxpy) -> qaxpy_k +#define QAXPYC_K OPENBLAS_DISPATCH(qaxpy) -> qaxpy_k +#define QCOPY_K OPENBLAS_DISPATCH(qcopy) -> qcopy_k +#define QDOTU_K OPENBLAS_DISPATCH(qdot) -> qdot_k +#define QDOTC_K OPENBLAS_DISPATCH(qdot) -> qdot_k +#define QNRM2_K OPENBLAS_DISPATCH(qnrm2) -> qnrm2_k +#define QSCAL_K OPENBLAS_DISPATCH(qscal) -> qscal_k +#define QSUM_K OPENBLAS_DISPATCH(qsum) -> qsum_k +#define QSWAP_K OPENBLAS_DISPATCH(qswap) -> qswap_k +#define QROT_K OPENBLAS_DISPATCH(qrot) -> qrot_k +#define QROTM_K OPENBLAS_DISPATCH(qrotm) -> qrotm_k + +#define QGEMV_N OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_T OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_R OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_C OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_O OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_U OPENBLAS_DISPATCH(qgemv) -> qgemv_t +#define QGEMV_S OPENBLAS_DISPATCH(qgemv) -> qgemv_n +#define QGEMV_D OPENBLAS_DISPATCH(qgemv) -> qgemv_t + +#define QGERU_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERC_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERV_K OPENBLAS_DISPATCH(qger) -> qger_k +#define QGERD_K OPENBLAS_DISPATCH(qger) -> qger_k + +#define QSYMV_U OPENBLAS_DISPATCH(qsymv) -> qsymv_U +#define QSYMV_L OPENBLAS_DISPATCH(qsymv) -> qsymv_L #define QSYMV_THREAD_U qsymv_thread_U #define QSYMV_THREAD_L qsymv_thread_L -#define QGEMM_ONCOPY gotoblas -> qgemm_oncopy -#define QGEMM_OTCOPY gotoblas -> qgemm_otcopy -#define QGEMM_INCOPY gotoblas -> qgemm_incopy -#define QGEMM_ITCOPY gotoblas -> qgemm_itcopy - -#define QTRMM_OUNUCOPY gotoblas -> qtrmm_ounucopy -#define QTRMM_OUTUCOPY gotoblas -> qtrmm_outucopy -#define QTRMM_OLNUCOPY gotoblas -> qtrmm_olnucopy -#define QTRMM_OLTUCOPY gotoblas -> qtrmm_oltucopy -#define QTRSM_OUNUCOPY gotoblas -> qtrsm_ounucopy -#define QTRSM_OUTUCOPY gotoblas -> qtrsm_outucopy -#define QTRSM_OLNUCOPY gotoblas -> qtrsm_olnucopy -#define QTRSM_OLTUCOPY gotoblas -> qtrsm_oltucopy - -#define QTRMM_IUNUCOPY gotoblas -> qtrmm_iunucopy -#define QTRMM_IUTUCOPY gotoblas -> qtrmm_iutucopy -#define QTRMM_ILNUCOPY gotoblas -> qtrmm_ilnucopy -#define QTRMM_ILTUCOPY gotoblas -> qtrmm_iltucopy -#define QTRSM_IUNUCOPY gotoblas -> qtrsm_iunucopy -#define QTRSM_IUTUCOPY gotoblas -> qtrsm_iutucopy -#define QTRSM_ILNUCOPY gotoblas -> qtrsm_ilnucopy -#define QTRSM_ILTUCOPY gotoblas -> qtrsm_iltucopy - -#define QTRMM_OUNNCOPY gotoblas -> qtrmm_ounncopy -#define QTRMM_OUTNCOPY gotoblas -> qtrmm_outncopy -#define QTRMM_OLNNCOPY gotoblas -> qtrmm_olnncopy -#define QTRMM_OLTNCOPY gotoblas -> qtrmm_oltncopy -#define QTRSM_OUNNCOPY gotoblas -> qtrsm_ounncopy -#define QTRSM_OUTNCOPY gotoblas -> qtrsm_outncopy -#define QTRSM_OLNNCOPY gotoblas -> qtrsm_olnncopy -#define QTRSM_OLTNCOPY gotoblas -> qtrsm_oltncopy - -#define QTRMM_IUNNCOPY gotoblas -> qtrmm_iunncopy -#define QTRMM_IUTNCOPY gotoblas -> qtrmm_iutncopy -#define QTRMM_ILNNCOPY gotoblas -> qtrmm_ilnncopy -#define QTRMM_ILTNCOPY gotoblas -> qtrmm_iltncopy -#define QTRSM_IUNNCOPY gotoblas -> qtrsm_iunncopy -#define QTRSM_IUTNCOPY gotoblas -> qtrsm_iutncopy -#define QTRSM_ILNNCOPY gotoblas -> qtrsm_ilnncopy -#define QTRSM_ILTNCOPY gotoblas -> qtrsm_iltncopy - -#define QGEMM_BETA gotoblas -> qgemm_beta -#define QGEMM_KERNEL gotoblas -> qgemm_kernel - -#define QTRMM_KERNEL_LN gotoblas -> qtrmm_kernel_LN -#define QTRMM_KERNEL_LT gotoblas -> qtrmm_kernel_LT -#define QTRMM_KERNEL_LR gotoblas -> qtrmm_kernel_LN -#define QTRMM_KERNEL_LC gotoblas -> qtrmm_kernel_LT -#define QTRMM_KERNEL_RN gotoblas -> qtrmm_kernel_RN -#define QTRMM_KERNEL_RT gotoblas -> qtrmm_kernel_RT -#define QTRMM_KERNEL_RR gotoblas -> qtrmm_kernel_RN -#define QTRMM_KERNEL_RC gotoblas -> qtrmm_kernel_RT - -#define QTRSM_KERNEL_LN gotoblas -> qtrsm_kernel_LN -#define QTRSM_KERNEL_LT gotoblas -> qtrsm_kernel_LT -#define QTRSM_KERNEL_LR gotoblas -> qtrsm_kernel_LN -#define QTRSM_KERNEL_LC gotoblas -> qtrsm_kernel_LT -#define QTRSM_KERNEL_RN gotoblas -> qtrsm_kernel_RN -#define QTRSM_KERNEL_RT gotoblas -> qtrsm_kernel_RT -#define QTRSM_KERNEL_RR gotoblas -> qtrsm_kernel_RN -#define QTRSM_KERNEL_RC gotoblas -> qtrsm_kernel_RT - -#define QSYMM_IUTCOPY gotoblas -> qsymm_iutcopy -#define QSYMM_ILTCOPY gotoblas -> qsymm_iltcopy -#define QSYMM_OUTCOPY gotoblas -> qsymm_outcopy -#define QSYMM_OLTCOPY gotoblas -> qsymm_oltcopy - -#define QNEG_TCOPY gotoblas -> qneg_tcopy -#define QLASWP_NCOPY gotoblas -> qlaswp_ncopy +#define QGEMM_ONCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_oncopy +#define QGEMM_OTCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_otcopy +#define QGEMM_INCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_incopy +#define QGEMM_ITCOPY OPENBLAS_DISPATCH(qgemm) -> qgemm_itcopy + +#define QTRMM_OUNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ounucopy +#define QTRMM_OUTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_outucopy +#define QTRMM_OLNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_olnucopy +#define QTRMM_OLTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_oltucopy +#define QTRSM_OUNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ounucopy +#define QTRSM_OUTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_outucopy +#define QTRSM_OLNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_olnucopy +#define QTRSM_OLTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_oltucopy + +#define QTRMM_IUNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iunucopy +#define QTRMM_IUTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iutucopy +#define QTRMM_ILNUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ilnucopy +#define QTRMM_ILTUCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iltucopy +#define QTRSM_IUNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iunucopy +#define QTRSM_IUTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iutucopy +#define QTRSM_ILNUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ilnucopy +#define QTRSM_ILTUCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iltucopy + +#define QTRMM_OUNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ounncopy +#define QTRMM_OUTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_outncopy +#define QTRMM_OLNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_olnncopy +#define QTRMM_OLTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_oltncopy +#define QTRSM_OUNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ounncopy +#define QTRSM_OUTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_outncopy +#define QTRSM_OLNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_olnncopy +#define QTRSM_OLTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_oltncopy + +#define QTRMM_IUNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iunncopy +#define QTRMM_IUTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iutncopy +#define QTRMM_ILNNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_ilnncopy +#define QTRMM_ILTNCOPY OPENBLAS_DISPATCH(qtrmm) -> qtrmm_iltncopy +#define QTRSM_IUNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iunncopy +#define QTRSM_IUTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iutncopy +#define QTRSM_ILNNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_ilnncopy +#define QTRSM_ILTNCOPY OPENBLAS_DISPATCH(qtrsm) -> qtrsm_iltncopy + +#define QGEMM_BETA OPENBLAS_DISPATCH(qgemm) -> qgemm_beta +#define QGEMM_KERNEL OPENBLAS_DISPATCH(qgemm) -> qgemm_kernel + +#define QTRMM_KERNEL_LN OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LN +#define QTRMM_KERNEL_LT OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LT +#define QTRMM_KERNEL_LR OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LN +#define QTRMM_KERNEL_LC OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_LT +#define QTRMM_KERNEL_RN OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RN +#define QTRMM_KERNEL_RT OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RT +#define QTRMM_KERNEL_RR OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RN +#define QTRMM_KERNEL_RC OPENBLAS_DISPATCH(qtrmm) -> qtrmm_kernel_RT + +#define QTRSM_KERNEL_LN OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LN +#define QTRSM_KERNEL_LT OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LT +#define QTRSM_KERNEL_LR OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LN +#define QTRSM_KERNEL_LC OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_LT +#define QTRSM_KERNEL_RN OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RN +#define QTRSM_KERNEL_RT OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RT +#define QTRSM_KERNEL_RR OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RN +#define QTRSM_KERNEL_RC OPENBLAS_DISPATCH(qtrsm) -> qtrsm_kernel_RT + +#define QSYMM_IUTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_iutcopy +#define QSYMM_ILTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_iltcopy +#define QSYMM_OUTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_outcopy +#define QSYMM_OLTCOPY OPENBLAS_DISPATCH(qsymm) -> qsymm_oltcopy + +#define QNEG_TCOPY OPENBLAS_DISPATCH(qneg) -> qneg_tcopy +#define QLASWP_NCOPY OPENBLAS_DISPATCH(qlaswp) -> qlaswp_ncopy #endif diff --git a/common_s.h b/common_s.h index 06b25b89fd..abf8c7e738 100644 --- a/common_s.h +++ b/common_s.h @@ -231,177 +231,177 @@ #else -#define SAMAX_K gotoblas -> samax_k -#define SAMIN_K gotoblas -> samin_k -#define SMAX_K gotoblas -> smax_k -#define SMIN_K gotoblas -> smin_k -#define ISAMAX_K gotoblas -> isamax_k -#define ISAMIN_K gotoblas -> isamin_k -#define ISMAX_K gotoblas -> ismax_k -#define ISMIN_K gotoblas -> ismin_k -#define SASUM_K gotoblas -> sasum_k -#define SSUM_K gotoblas -> ssum_k -#define SAXPYU_K gotoblas -> saxpy_k -#define SAXPYC_K gotoblas -> saxpy_k -#define SCOPY_K gotoblas -> scopy_k -#define SDOTU_K gotoblas -> sdot_k -#define SDOTC_K gotoblas -> sdot_k -#define SDSDOT_K gotoblas -> dsdot_k -#define DSDOT_K gotoblas -> dsdot_k -#define SNRM2_K gotoblas -> snrm2_k -#define SSCAL_K gotoblas -> sscal_k -#define SSWAP_K gotoblas -> sswap_k -#define SROT_K gotoblas -> srot_k -#define SROTM_K gotoblas -> srotm_k - -#define SGEMV_N gotoblas -> sgemv_n -#define SGEMV_T gotoblas -> sgemv_t -#define SGEMV_R gotoblas -> sgemv_n -#define SGEMV_C gotoblas -> sgemv_t -#define SGEMV_O gotoblas -> sgemv_n -#define SGEMV_U gotoblas -> sgemv_t -#define SGEMV_S gotoblas -> sgemv_n -#define SGEMV_D gotoblas -> sgemv_t - -#define SGERU_K gotoblas -> sger_k -#define SGERC_K gotoblas -> sger_k -#define SGERV_K gotoblas -> sger_k -#define SGERD_K gotoblas -> sger_k - -#define SSYMV_U gotoblas -> ssymv_U -#define SSYMV_L gotoblas -> ssymv_L +#define SAMAX_K OPENBLAS_DISPATCH(samax) -> samax_k +#define SAMIN_K OPENBLAS_DISPATCH(samin) -> samin_k +#define SMAX_K OPENBLAS_DISPATCH(smax) -> smax_k +#define SMIN_K OPENBLAS_DISPATCH(smin) -> smin_k +#define ISAMAX_K OPENBLAS_DISPATCH(isamax) -> isamax_k +#define ISAMIN_K OPENBLAS_DISPATCH(isamin) -> isamin_k +#define ISMAX_K OPENBLAS_DISPATCH(ismax) -> ismax_k +#define ISMIN_K OPENBLAS_DISPATCH(ismin) -> ismin_k +#define SASUM_K OPENBLAS_DISPATCH(sasum) -> sasum_k +#define SSUM_K OPENBLAS_DISPATCH(ssum) -> ssum_k +#define SAXPYU_K OPENBLAS_DISPATCH(saxpy) -> saxpy_k +#define SAXPYC_K OPENBLAS_DISPATCH(saxpy) -> saxpy_k +#define SCOPY_K OPENBLAS_DISPATCH(scopy) -> scopy_k +#define SDOTU_K OPENBLAS_DISPATCH(sdot) -> sdot_k +#define SDOTC_K OPENBLAS_DISPATCH(sdot) -> sdot_k +#define SDSDOT_K OPENBLAS_DISPATCH(dsdot) -> dsdot_k +#define DSDOT_K OPENBLAS_DISPATCH(dsdot) -> dsdot_k +#define SNRM2_K OPENBLAS_DISPATCH(snrm2) -> snrm2_k +#define SSCAL_K OPENBLAS_DISPATCH(sscal) -> sscal_k +#define SSWAP_K OPENBLAS_DISPATCH(sswap) -> sswap_k +#define SROT_K OPENBLAS_DISPATCH(srot) -> srot_k +#define SROTM_K OPENBLAS_DISPATCH(srotm) -> srotm_k + +#define SGEMV_N OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_T OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_R OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_C OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_O OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_U OPENBLAS_DISPATCH(sgemv) -> sgemv_t +#define SGEMV_S OPENBLAS_DISPATCH(sgemv) -> sgemv_n +#define SGEMV_D OPENBLAS_DISPATCH(sgemv) -> sgemv_t + +#define SGERU_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERC_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERV_K OPENBLAS_DISPATCH(sger) -> sger_k +#define SGERD_K OPENBLAS_DISPATCH(sger) -> sger_k + +#define SSYMV_U OPENBLAS_DISPATCH(ssymv) -> ssymv_U +#define SSYMV_L OPENBLAS_DISPATCH(ssymv) -> ssymv_L #define SSYMV_THREAD_U ssymv_thread_U #define SSYMV_THREAD_L ssymv_thread_L #ifdef ARCH_X86_64 -#define SGEMM_DIRECT_PERFORMANT gotoblas -> sgemm_direct_performant -#define SGEMM_DIRECT gotoblas -> sgemm_direct +#define SGEMM_DIRECT_PERFORMANT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_performant +#define SGEMM_DIRECT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct #elif ARCH_ARM64 -#define SGEMM_DIRECT_PERFORMANT gotoblas -> sgemm_direct_performant -#define SGEMM_DIRECT gotoblas -> sgemm_direct -#define SGEMM_DIRECT_ALPHA_BETA gotoblas -> sgemm_direct_alpha_beta -#define SSYMM_DIRECT_ALPHA_BETA_LU gotoblas -> ssymm_direct_alpha_betaLU -#define SSYMM_DIRECT_ALPHA_BETA_LL gotoblas -> ssymm_direct_alpha_betaLL -#define STRMM_DIRECT_LNUN gotoblas -> strmm_direct_LNUN -#define STRMM_DIRECT_LNLN gotoblas -> strmm_direct_LNLN -#define STRMM_DIRECT_LTUN gotoblas -> strmm_direct_LTUN -#define STRMM_DIRECT_LTLN gotoblas -> strmm_direct_LTLN -#define SSYRK_DIRECT_ALPHA_BETA_UN gotoblas -> ssyrk_direct_alpha_betaUN -#define SSYRK_DIRECT_ALPHA_BETA_UT gotoblas -> ssyrk_direct_alpha_betaUT -#define SSYRK_DIRECT_ALPHA_BETA_LN gotoblas -> ssyrk_direct_alpha_betaLN -#define SSYRK_DIRECT_ALPHA_BETA_LT gotoblas -> ssyrk_direct_alpha_betaLT -#define SSYR2K_DIRECT_ALPHA_BETA_UN gotoblas -> ssyr2k_direct_alpha_betaUN -#define SSYR2K_DIRECT_ALPHA_BETA_UT gotoblas -> ssyr2k_direct_alpha_betaUT -#define SSYR2K_DIRECT_ALPHA_BETA_LN gotoblas -> ssyr2k_direct_alpha_betaLN -#define SSYR2K_DIRECT_ALPHA_BETA_LT gotoblas -> ssyr2k_direct_alpha_betaLT -#define SME_SGEMM_KERNEL gotoblas -> sme_sgemm_kernel +#define SGEMM_DIRECT_PERFORMANT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_performant +#define SGEMM_DIRECT OPENBLAS_DISPATCH(sgemm) -> sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA OPENBLAS_DISPATCH(sgemm) -> sgemm_direct_alpha_beta +#define SSYMM_DIRECT_ALPHA_BETA_LU OPENBLAS_DISPATCH(ssymm) -> ssymm_direct_alpha_betaLU +#define SSYMM_DIRECT_ALPHA_BETA_LL OPENBLAS_DISPATCH(ssymm) -> ssymm_direct_alpha_betaLL +#define STRMM_DIRECT_LNUN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LNUN +#define STRMM_DIRECT_LNLN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LNLN +#define STRMM_DIRECT_LTUN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LTUN +#define STRMM_DIRECT_LTLN OPENBLAS_DISPATCH(strmm) -> strmm_direct_LTLN +#define SSYRK_DIRECT_ALPHA_BETA_UN OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaUN +#define SSYRK_DIRECT_ALPHA_BETA_UT OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaUT +#define SSYRK_DIRECT_ALPHA_BETA_LN OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaLN +#define SSYRK_DIRECT_ALPHA_BETA_LT OPENBLAS_DISPATCH(ssyrk) -> ssyrk_direct_alpha_betaLT +#define SSYR2K_DIRECT_ALPHA_BETA_UN OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaUN +#define SSYR2K_DIRECT_ALPHA_BETA_UT OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaUT +#define SSYR2K_DIRECT_ALPHA_BETA_LN OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaLN +#define SSYR2K_DIRECT_ALPHA_BETA_LT OPENBLAS_DISPATCH(ssyr2k) -> ssyr2k_direct_alpha_betaLT +#define SME_SGEMM_KERNEL OPENBLAS_DISPATCH(sgemm) -> sme_sgemm_kernel #endif -#define SGEMM_ONCOPY gotoblas -> sgemm_oncopy -#define SGEMM_OTCOPY gotoblas -> sgemm_otcopy -#define SGEMM_INCOPY gotoblas -> sgemm_incopy -#define SGEMM_ITCOPY gotoblas -> sgemm_itcopy - -#define STRMM_OUNUCOPY gotoblas -> strmm_ounucopy -#define STRMM_OUTUCOPY gotoblas -> strmm_outucopy -#define STRMM_OLNUCOPY gotoblas -> strmm_olnucopy -#define STRMM_OLTUCOPY gotoblas -> strmm_oltucopy -#define STRSM_OUNUCOPY gotoblas -> strsm_ounucopy -#define STRSM_OUTUCOPY gotoblas -> strsm_outucopy -#define STRSM_OLNUCOPY gotoblas -> strsm_olnucopy -#define STRSM_OLTUCOPY gotoblas -> strsm_oltucopy - -#define STRMM_IUNUCOPY gotoblas -> strmm_iunucopy -#define STRMM_IUTUCOPY gotoblas -> strmm_iutucopy -#define STRMM_ILNUCOPY gotoblas -> strmm_ilnucopy -#define STRMM_ILTUCOPY gotoblas -> strmm_iltucopy -#define STRSM_IUNUCOPY gotoblas -> strsm_iunucopy -#define STRSM_IUTUCOPY gotoblas -> strsm_iutucopy -#define STRSM_ILNUCOPY gotoblas -> strsm_ilnucopy -#define STRSM_ILTUCOPY gotoblas -> strsm_iltucopy - -#define STRMM_OUNNCOPY gotoblas -> strmm_ounncopy -#define STRMM_OUTNCOPY gotoblas -> strmm_outncopy -#define STRMM_OLNNCOPY gotoblas -> strmm_olnncopy -#define STRMM_OLTNCOPY gotoblas -> strmm_oltncopy -#define STRSM_OUNNCOPY gotoblas -> strsm_ounncopy -#define STRSM_OUTNCOPY gotoblas -> strsm_outncopy -#define STRSM_OLNNCOPY gotoblas -> strsm_olnncopy -#define STRSM_OLTNCOPY gotoblas -> strsm_oltncopy - -#define STRMM_IUNNCOPY gotoblas -> strmm_iunncopy -#define STRMM_IUTNCOPY gotoblas -> strmm_iutncopy -#define STRMM_ILNNCOPY gotoblas -> strmm_ilnncopy -#define STRMM_ILTNCOPY gotoblas -> strmm_iltncopy -#define SSYMM_INCOPY gotoblas -> ssymm_incopy -#define SSYMM_ITCOPY gotoblas -> ssymm_itcopy -#define STRMM_INCOPY gotoblas -> strmm_incopy -#define STRMM_ITCOPY gotoblas -> strmm_itcopy - -#define STRSM_IUNNCOPY gotoblas -> strsm_iunncopy -#define STRSM_IUTNCOPY gotoblas -> strsm_iutncopy -#define STRSM_ILNNCOPY gotoblas -> strsm_ilnncopy -#define STRSM_ILTNCOPY gotoblas -> strsm_iltncopy - -#define SGEMM_BETA gotoblas -> sgemm_beta -#define SGEMM_KERNEL gotoblas -> sgemm_kernel -#define SSYMM_KERNEL gotoblas -> ssymm_kernel -#define STRMM_GEMM_KERNEL gotoblas -> strmm_gemm_kernel - -#define STRMM_KERNEL_LN gotoblas -> strmm_kernel_LN -#define STRMM_KERNEL_LT gotoblas -> strmm_kernel_LT -#define STRMM_KERNEL_LR gotoblas -> strmm_kernel_LN -#define STRMM_KERNEL_LC gotoblas -> strmm_kernel_LT -#define STRMM_KERNEL_RN gotoblas -> strmm_kernel_RN -#define STRMM_KERNEL_RT gotoblas -> strmm_kernel_RT -#define STRMM_KERNEL_RR gotoblas -> strmm_kernel_RN -#define STRMM_KERNEL_RC gotoblas -> strmm_kernel_RT - -#define STRSM_KERNEL_LN gotoblas -> strsm_kernel_LN -#define STRSM_KERNEL_LT gotoblas -> strsm_kernel_LT -#define STRSM_KERNEL_LR gotoblas -> strsm_kernel_LN -#define STRSM_KERNEL_LC gotoblas -> strsm_kernel_LT -#define STRSM_KERNEL_RN gotoblas -> strsm_kernel_RN -#define STRSM_KERNEL_RT gotoblas -> strsm_kernel_RT -#define STRSM_KERNEL_RR gotoblas -> strsm_kernel_RN -#define STRSM_KERNEL_RC gotoblas -> strsm_kernel_RT - -#define SSYMM_IUTCOPY gotoblas -> ssymm_iutcopy -#define SSYMM_ILTCOPY gotoblas -> ssymm_iltcopy -#define SSYMM_OUTCOPY gotoblas -> ssymm_outcopy -#define SSYMM_OLTCOPY gotoblas -> ssymm_oltcopy - -#define SNEG_TCOPY gotoblas -> sneg_tcopy -#define SLASWP_NCOPY gotoblas -> slaswp_ncopy - -#define SAXPBY_K gotoblas -> saxpby_k - -#define SOMATCOPY_K_CN gotoblas -> somatcopy_k_cn -#define SOMATCOPY_K_RN gotoblas -> somatcopy_k_rn -#define SOMATCOPY_K_CT gotoblas -> somatcopy_k_ct -#define SOMATCOPY_K_RT gotoblas -> somatcopy_k_rt -#define SIMATCOPY_K_CN gotoblas -> simatcopy_k_cn -#define SIMATCOPY_K_RN gotoblas -> simatcopy_k_rn -#define SIMATCOPY_K_CT gotoblas -> simatcopy_k_ct -#define SIMATCOPY_K_RT gotoblas -> simatcopy_k_rt - -#define SGEADD_K gotoblas -> sgeadd_k - -#define SGEMM_SMALL_MATRIX_PERMIT gotoblas -> sgemm_small_matrix_permit +#define SGEMM_ONCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_oncopy +#define SGEMM_OTCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_otcopy +#define SGEMM_INCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_incopy +#define SGEMM_ITCOPY OPENBLAS_DISPATCH(sgemm) -> sgemm_itcopy + +#define STRMM_OUNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ounucopy +#define STRMM_OUTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_outucopy +#define STRMM_OLNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_olnucopy +#define STRMM_OLTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_oltucopy +#define STRSM_OUNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ounucopy +#define STRSM_OUTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_outucopy +#define STRSM_OLNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_olnucopy +#define STRSM_OLTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_oltucopy + +#define STRMM_IUNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iunucopy +#define STRMM_IUTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iutucopy +#define STRMM_ILNUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ilnucopy +#define STRMM_ILTUCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iltucopy +#define STRSM_IUNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iunucopy +#define STRSM_IUTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iutucopy +#define STRSM_ILNUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ilnucopy +#define STRSM_ILTUCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iltucopy + +#define STRMM_OUNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ounncopy +#define STRMM_OUTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_outncopy +#define STRMM_OLNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_olnncopy +#define STRMM_OLTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_oltncopy +#define STRSM_OUNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ounncopy +#define STRSM_OUTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_outncopy +#define STRSM_OLNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_olnncopy +#define STRSM_OLTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_oltncopy + +#define STRMM_IUNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iunncopy +#define STRMM_IUTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iutncopy +#define STRMM_ILNNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_ilnncopy +#define STRMM_ILTNCOPY OPENBLAS_DISPATCH(strmm) -> strmm_iltncopy +#define SSYMM_INCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_incopy +#define SSYMM_ITCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_itcopy +#define STRMM_INCOPY OPENBLAS_DISPATCH(strmm) -> strmm_incopy +#define STRMM_ITCOPY OPENBLAS_DISPATCH(strmm) -> strmm_itcopy + +#define STRSM_IUNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iunncopy +#define STRSM_IUTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iutncopy +#define STRSM_ILNNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_ilnncopy +#define STRSM_ILTNCOPY OPENBLAS_DISPATCH(strsm) -> strsm_iltncopy + +#define SGEMM_BETA OPENBLAS_DISPATCH(sgemm) -> sgemm_beta +#define SGEMM_KERNEL OPENBLAS_DISPATCH(sgemm) -> sgemm_kernel +#define SSYMM_KERNEL OPENBLAS_DISPATCH(ssymm) -> ssymm_kernel +#define STRMM_GEMM_KERNEL OPENBLAS_DISPATCH(strmm) -> strmm_gemm_kernel + +#define STRMM_KERNEL_LN OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LN +#define STRMM_KERNEL_LT OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LT +#define STRMM_KERNEL_LR OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LN +#define STRMM_KERNEL_LC OPENBLAS_DISPATCH(strmm) -> strmm_kernel_LT +#define STRMM_KERNEL_RN OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RN +#define STRMM_KERNEL_RT OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RT +#define STRMM_KERNEL_RR OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RN +#define STRMM_KERNEL_RC OPENBLAS_DISPATCH(strmm) -> strmm_kernel_RT + +#define STRSM_KERNEL_LN OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LN +#define STRSM_KERNEL_LT OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LT +#define STRSM_KERNEL_LR OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LN +#define STRSM_KERNEL_LC OPENBLAS_DISPATCH(strsm) -> strsm_kernel_LT +#define STRSM_KERNEL_RN OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RN +#define STRSM_KERNEL_RT OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RT +#define STRSM_KERNEL_RR OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RN +#define STRSM_KERNEL_RC OPENBLAS_DISPATCH(strsm) -> strsm_kernel_RT + +#define SSYMM_IUTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_iutcopy +#define SSYMM_ILTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_iltcopy +#define SSYMM_OUTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_outcopy +#define SSYMM_OLTCOPY OPENBLAS_DISPATCH(ssymm) -> ssymm_oltcopy + +#define SNEG_TCOPY OPENBLAS_DISPATCH(sneg) -> sneg_tcopy +#define SLASWP_NCOPY OPENBLAS_DISPATCH(slaswp) -> slaswp_ncopy + +#define SAXPBY_K OPENBLAS_DISPATCH(saxpby) -> saxpby_k + +#define SOMATCOPY_K_CN OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_cn +#define SOMATCOPY_K_RN OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_rn +#define SOMATCOPY_K_CT OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_ct +#define SOMATCOPY_K_RT OPENBLAS_DISPATCH(somatcopy) -> somatcopy_k_rt +#define SIMATCOPY_K_CN OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_cn +#define SIMATCOPY_K_RN OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_rn +#define SIMATCOPY_K_CT OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_ct +#define SIMATCOPY_K_RT OPENBLAS_DISPATCH(simatcopy) -> simatcopy_k_rt + +#define SGEADD_K OPENBLAS_DISPATCH(sgeadd) -> sgeadd_k + +#define SGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(sgemm) -> sgemm_small_matrix_permit #endif -#define SGEMM_SMALL_KERNEL_BASE FUNC_BASE(sgemm_small_kernel_nn) -#define SGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sgemm_small_kernel_nn) -#define SGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sgemm_small_kernel_nt) -#define SGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sgemm_small_kernel_tn) -#define SGEMM_SMALL_KERNEL_TT FUNC_OFFSET(sgemm_small_kernel_tt) +#define SGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(sgemm) +#define SGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_nn) +#define SGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_nt) +#define SGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_tn) +#define SGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_tt) -#define SGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(sgemm_small_kernel_b0_nn) -#define SGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(sgemm_small_kernel_b0_nt) -#define SGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(sgemm_small_kernel_b0_tn) -#define SGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(sgemm_small_kernel_b0_tt) +#define SGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_nn) +#define SGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_nt) +#define SGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_tn) +#define SGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(sgemm, sgemm_small_kernel_b0_tt) #define SGEMM_NN sgemm_nn diff --git a/common_sb.h b/common_sb.h index d79c3e0cbc..9e51a2d39b 100644 --- a/common_sb.h +++ b/common_sb.h @@ -27,34 +27,34 @@ #define SBGEMM_SMALL_MATRIX_PERMIT sbgemm_small_matrix_permit #else -#define SBDOT_K gotoblas -> sbdot_k -#define SBSTOBF16_K gotoblas -> sbstobf16_k -#define SBDTOBF16_K gotoblas -> sbdtobf16_k -#define SBF16TOS_K gotoblas -> sbf16tos_k -#define DBF16TOD_K gotoblas -> dbf16tod_k -#define SBGEMV_N_K gotoblas -> sbgemv_n -#define SBGEMV_T_K gotoblas -> sbgemv_t +#define SBDOT_K OPENBLAS_DISPATCH(sbdot) -> sbdot_k +#define SBSTOBF16_K OPENBLAS_DISPATCH(sbstobf16) -> sbstobf16_k +#define SBDTOBF16_K OPENBLAS_DISPATCH(sbdtobf16) -> sbdtobf16_k +#define SBF16TOS_K OPENBLAS_DISPATCH(sbf16tos) -> sbf16tos_k +#define DBF16TOD_K OPENBLAS_DISPATCH(dbf16tod) -> dbf16tod_k +#define SBGEMV_N_K OPENBLAS_DISPATCH(sbgemv) -> sbgemv_n +#define SBGEMV_T_K OPENBLAS_DISPATCH(sbgemv) -> sbgemv_t -#define SBGEMM_ONCOPY gotoblas -> sbgemm_oncopy -#define SBGEMM_OTCOPY gotoblas -> sbgemm_otcopy -#define SBGEMM_INCOPY gotoblas -> sbgemm_incopy -#define SBGEMM_ITCOPY gotoblas -> sbgemm_itcopy -#define SBGEMM_BETA gotoblas -> sbgemm_beta -#define SBGEMM_KERNEL gotoblas -> sbgemm_kernel +#define SBGEMM_ONCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_oncopy +#define SBGEMM_OTCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_otcopy +#define SBGEMM_INCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_incopy +#define SBGEMM_ITCOPY OPENBLAS_DISPATCH(sbgemm) -> sbgemm_itcopy +#define SBGEMM_BETA OPENBLAS_DISPATCH(sbgemm) -> sbgemm_beta +#define SBGEMM_KERNEL OPENBLAS_DISPATCH(sbgemm) -> sbgemm_kernel -#define SBGEMM_SMALL_MATRIX_PERMIT gotoblas -> sbgemm_small_matrix_permit +#define SBGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(sbgemm) -> sbgemm_small_matrix_permit #endif -#define SBGEMM_SMALL_KERNEL_BASE FUNC_BASE(sbgemm_small_kernel_nn) -#define SBGEMM_SMALL_KERNEL_NN FUNC_OFFSET(sbgemm_small_kernel_nn) -#define SBGEMM_SMALL_KERNEL_NT FUNC_OFFSET(sbgemm_small_kernel_nt) -#define SBGEMM_SMALL_KERNEL_TN FUNC_OFFSET(sbgemm_small_kernel_tn) -#define SBGEMM_SMALL_KERNEL_TT FUNC_OFFSET(sbgemm_small_kernel_tt) +#define SBGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(sbgemm) +#define SBGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_nn) +#define SBGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_nt) +#define SBGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_tn) +#define SBGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_tt) -#define SBGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(sbgemm_small_kernel_b0_nn) -#define SBGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(sbgemm_small_kernel_b0_nt) -#define SBGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(sbgemm_small_kernel_b0_tn) -#define SBGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(sbgemm_small_kernel_b0_tt) +#define SBGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_nn) +#define SBGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_nt) +#define SBGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_tn) +#define SBGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(sbgemm, sbgemm_small_kernel_b0_tt) #define SBGEMM_NN sbgemm_nn #define SBGEMM_CN sbgemm_tn diff --git a/common_sh.h b/common_sh.h index 99dbb65180..7095b7d938 100644 --- a/common_sh.h +++ b/common_sh.h @@ -51,21 +51,21 @@ #else // #DYNAMIC_ARCH -#define SHGEMM_ONCOPY gotoblas -> shgemm_oncopy -#define SHGEMM_OTCOPY gotoblas -> shgemm_otcopy +#define SHGEMM_ONCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_oncopy +#define SHGEMM_OTCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_otcopy #if SGEMM_DEFAULT_UNROLL_M == SGEMM_DEFAULT_UNROLL_N -#define SHGEMM_INCOPY gotoblas -> shgemm_oncopy -#define SHGEMM_ITCOPY gotoblas -> shgemm_otcopy +#define SHGEMM_INCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_oncopy +#define SHGEMM_ITCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_otcopy #else -#define SHGEMM_INCOPY gotoblas -> shgemm_incopy -#define SHGEMM_ITCOPY gotoblas -> shgemm_itcopy +#define SHGEMM_INCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_incopy +#define SHGEMM_ITCOPY OPENBLAS_DISPATCH(shgemm) -> shgemm_itcopy #endif -#define SHGEMM_BETA gotoblas -> shgemm_beta -#define SHGEMM_KERNEL gotoblas -> shgemm_kernel +#define SHGEMM_BETA OPENBLAS_DISPATCH(shgemm) -> shgemm_beta +#define SHGEMM_KERNEL OPENBLAS_DISPATCH(shgemm) -> shgemm_kernel -#define SHGEMV_N_K gotoblas->shgemv_n -#define SHGEMV_T_K gotoblas->shgemv_t +#define SHGEMV_N_K OPENBLAS_DISPATCH(shgemv)->shgemv_n +#define SHGEMV_T_K OPENBLAS_DISPATCH(shgemv)->shgemv_t #endif // #DYNAMIC_ARCH diff --git a/common_x.h b/common_x.h index 12ed909c57..899d367b2a 100644 --- a/common_x.h +++ b/common_x.h @@ -212,46 +212,46 @@ #else -#define XAMAX_K gotoblas -> xamax_k -#define XAMIN_K gotoblas -> xamin_k +#define XAMAX_K OPENBLAS_DISPATCH(xamax) -> xamax_k +#define XAMIN_K OPENBLAS_DISPATCH(xamin) -> xamin_k #define XMAX_K gotoblas -> xmax_k #define XMIN_K gotoblas -> xmin_k -#define IXAMAX_K gotoblas -> ixamax_k -#define IXAMIN_K gotoblas -> ixamin_k +#define IXAMAX_K OPENBLAS_DISPATCH(ixamax) -> ixamax_k +#define IXAMIN_K OPENBLAS_DISPATCH(ixamin) -> ixamin_k #define IXMAX_K gotoblas -> ixmax_k #define IXMIN_K gotoblas -> ixmin_k -#define XASUM_K gotoblas -> xasum_k -#define XAXPYU_K gotoblas -> xaxpy_k -#define XAXPYC_K gotoblas -> xaxpyc_k -#define XCOPY_K gotoblas -> xcopy_k -#define XDOTU_K gotoblas -> xdotu_k -#define XDOTC_K gotoblas -> xdotc_k -#define XNRM2_K gotoblas -> xnrm2_k -#define XSCAL_K gotoblas -> xscal_k -#define XSUM_K gotoblas -> xsum_k -#define XSWAP_K gotoblas -> xswap_k -#define XROT_K gotoblas -> xqrot_k - -#define XGEMV_N gotoblas -> xgemv_n -#define XGEMV_T gotoblas -> xgemv_t -#define XGEMV_R gotoblas -> xgemv_r -#define XGEMV_C gotoblas -> xgemv_c -#define XGEMV_O gotoblas -> xgemv_o -#define XGEMV_U gotoblas -> xgemv_u -#define XGEMV_S gotoblas -> xgemv_s -#define XGEMV_D gotoblas -> xgemv_d - -#define XGERU_K gotoblas -> xgeru_k -#define XGERC_K gotoblas -> xgerc_k -#define XGERV_K gotoblas -> xgerv_k -#define XGERD_K gotoblas -> xgerd_k - -#define XSYMV_U gotoblas -> xsymv_U -#define XSYMV_L gotoblas -> xsymv_L -#define XHEMV_U gotoblas -> xhemv_U -#define XHEMV_L gotoblas -> xhemv_L -#define XHEMV_V gotoblas -> xhemv_V -#define XHEMV_M gotoblas -> xhemv_M +#define XASUM_K OPENBLAS_DISPATCH(xasum) -> xasum_k +#define XAXPYU_K OPENBLAS_DISPATCH(xaxpy) -> xaxpy_k +#define XAXPYC_K OPENBLAS_DISPATCH(xaxpyc) -> xaxpyc_k +#define XCOPY_K OPENBLAS_DISPATCH(xcopy) -> xcopy_k +#define XDOTU_K OPENBLAS_DISPATCH(xdotu) -> xdotu_k +#define XDOTC_K OPENBLAS_DISPATCH(xdotc) -> xdotc_k +#define XNRM2_K OPENBLAS_DISPATCH(xnrm2) -> xnrm2_k +#define XSCAL_K OPENBLAS_DISPATCH(xscal) -> xscal_k +#define XSUM_K OPENBLAS_DISPATCH(xsum) -> xsum_k +#define XSWAP_K OPENBLAS_DISPATCH(xswap) -> xswap_k +#define XROT_K OPENBLAS_DISPATCH(xqrot) -> xqrot_k + +#define XGEMV_N OPENBLAS_DISPATCH(xgemv) -> xgemv_n +#define XGEMV_T OPENBLAS_DISPATCH(xgemv) -> xgemv_t +#define XGEMV_R OPENBLAS_DISPATCH(xgemv) -> xgemv_r +#define XGEMV_C OPENBLAS_DISPATCH(xgemv) -> xgemv_c +#define XGEMV_O OPENBLAS_DISPATCH(xgemv) -> xgemv_o +#define XGEMV_U OPENBLAS_DISPATCH(xgemv) -> xgemv_u +#define XGEMV_S OPENBLAS_DISPATCH(xgemv) -> xgemv_s +#define XGEMV_D OPENBLAS_DISPATCH(xgemv) -> xgemv_d + +#define XGERU_K OPENBLAS_DISPATCH(xgeru) -> xgeru_k +#define XGERC_K OPENBLAS_DISPATCH(xgerc) -> xgerc_k +#define XGERV_K OPENBLAS_DISPATCH(xgerv) -> xgerv_k +#define XGERD_K OPENBLAS_DISPATCH(xgerd) -> xgerd_k + +#define XSYMV_U OPENBLAS_DISPATCH(xsymv) -> xsymv_U +#define XSYMV_L OPENBLAS_DISPATCH(xsymv) -> xsymv_L +#define XHEMV_U OPENBLAS_DISPATCH(xhemv) -> xhemv_U +#define XHEMV_L OPENBLAS_DISPATCH(xhemv) -> xhemv_L +#define XHEMV_V OPENBLAS_DISPATCH(xhemv) -> xhemv_V +#define XHEMV_M OPENBLAS_DISPATCH(xhemv) -> xhemv_M #define XSYMV_THREAD_U xsymv_thread_U #define XSYMV_THREAD_L xsymv_thread_L @@ -260,127 +260,127 @@ #define XHEMV_THREAD_V xhemv_thread_V #define XHEMV_THREAD_M xhemv_thread_M -#define XGEMM_ONCOPY gotoblas -> xgemm_oncopy -#define XGEMM_OTCOPY gotoblas -> xgemm_otcopy -#define XGEMM_INCOPY gotoblas -> xgemm_incopy -#define XGEMM_ITCOPY gotoblas -> xgemm_itcopy - -#define XTRMM_OUNUCOPY gotoblas -> xtrmm_ounucopy -#define XTRMM_OUTUCOPY gotoblas -> xtrmm_outucopy -#define XTRMM_OLNUCOPY gotoblas -> xtrmm_olnucopy -#define XTRMM_OLTUCOPY gotoblas -> xtrmm_oltucopy -#define XTRSM_OUNUCOPY gotoblas -> xtrsm_ounucopy -#define XTRSM_OUTUCOPY gotoblas -> xtrsm_outucopy -#define XTRSM_OLNUCOPY gotoblas -> xtrsm_olnucopy -#define XTRSM_OLTUCOPY gotoblas -> xtrsm_oltucopy - -#define XTRMM_IUNUCOPY gotoblas -> xtrmm_iunucopy -#define XTRMM_IUTUCOPY gotoblas -> xtrmm_iutucopy -#define XTRMM_ILNUCOPY gotoblas -> xtrmm_ilnucopy -#define XTRMM_ILTUCOPY gotoblas -> xtrmm_iltucopy -#define XTRSM_IUNUCOPY gotoblas -> xtrsm_iunucopy -#define XTRSM_IUTUCOPY gotoblas -> xtrsm_iutucopy -#define XTRSM_ILNUCOPY gotoblas -> xtrsm_ilnucopy -#define XTRSM_ILTUCOPY gotoblas -> xtrsm_iltucopy - -#define XTRMM_OUNNCOPY gotoblas -> xtrmm_ounncopy -#define XTRMM_OUTNCOPY gotoblas -> xtrmm_outncopy -#define XTRMM_OLNNCOPY gotoblas -> xtrmm_olnncopy -#define XTRMM_OLTNCOPY gotoblas -> xtrmm_oltncopy -#define XTRSM_OUNNCOPY gotoblas -> xtrsm_ounncopy -#define XTRSM_OUTNCOPY gotoblas -> xtrsm_outncopy -#define XTRSM_OLNNCOPY gotoblas -> xtrsm_olnncopy -#define XTRSM_OLTNCOPY gotoblas -> xtrsm_oltncopy - -#define XTRMM_IUNNCOPY gotoblas -> xtrmm_iunncopy -#define XTRMM_IUTNCOPY gotoblas -> xtrmm_iutncopy -#define XTRMM_ILNNCOPY gotoblas -> xtrmm_ilnncopy -#define XTRMM_ILTNCOPY gotoblas -> xtrmm_iltncopy -#define XTRSM_IUNNCOPY gotoblas -> xtrsm_iunncopy -#define XTRSM_IUTNCOPY gotoblas -> xtrsm_iutncopy -#define XTRSM_ILNNCOPY gotoblas -> xtrsm_ilnncopy -#define XTRSM_ILTNCOPY gotoblas -> xtrsm_iltncopy - -#define XGEMM_BETA gotoblas -> xgemm_beta -#define XGEMM_KERNEL_N gotoblas -> xgemm_kernel_n -#define XGEMM_KERNEL_L gotoblas -> xgemm_kernel_l -#define XGEMM_KERNEL_R gotoblas -> xgemm_kernel_r -#define XGEMM_KERNEL_B gotoblas -> xgemm_kernel_b - -#define XTRMM_KERNEL_LN gotoblas -> xtrmm_kernel_LN -#define XTRMM_KERNEL_LT gotoblas -> xtrmm_kernel_LT -#define XTRMM_KERNEL_LR gotoblas -> xtrmm_kernel_LR -#define XTRMM_KERNEL_LC gotoblas -> xtrmm_kernel_LC -#define XTRMM_KERNEL_RN gotoblas -> xtrmm_kernel_RN -#define XTRMM_KERNEL_RT gotoblas -> xtrmm_kernel_RT -#define XTRMM_KERNEL_RR gotoblas -> xtrmm_kernel_RR -#define XTRMM_KERNEL_RC gotoblas -> xtrmm_kernel_RC - -#define XTRSM_KERNEL_LN gotoblas -> xtrsm_kernel_LN -#define XTRSM_KERNEL_LT gotoblas -> xtrsm_kernel_LT -#define XTRSM_KERNEL_LR gotoblas -> xtrsm_kernel_LR -#define XTRSM_KERNEL_LC gotoblas -> xtrsm_kernel_LC -#define XTRSM_KERNEL_RN gotoblas -> xtrsm_kernel_RN -#define XTRSM_KERNEL_RT gotoblas -> xtrsm_kernel_RT -#define XTRSM_KERNEL_RR gotoblas -> xtrsm_kernel_RR -#define XTRSM_KERNEL_RC gotoblas -> xtrsm_kernel_RC - -#define XSYMM_IUTCOPY gotoblas -> xsymm_iutcopy -#define XSYMM_ILTCOPY gotoblas -> xsymm_iltcopy -#define XSYMM_OUTCOPY gotoblas -> xsymm_outcopy -#define XSYMM_OLTCOPY gotoblas -> xsymm_oltcopy - -#define XHEMM_OUTCOPY gotoblas -> xhemm_outcopy -#define XHEMM_OLTCOPY gotoblas -> xhemm_oltcopy -#define XHEMM_IUTCOPY gotoblas -> xhemm_iutcopy -#define XHEMM_ILTCOPY gotoblas -> xhemm_iltcopy - -#define XGEMM3M_ONCOPYB gotoblas -> xgemm3m_oncopyb -#define XGEMM3M_ONCOPYR gotoblas -> xgemm3m_oncopyr -#define XGEMM3M_ONCOPYI gotoblas -> xgemm3m_oncopyi -#define XGEMM3M_OTCOPYB gotoblas -> xgemm3m_otcopyb -#define XGEMM3M_OTCOPYR gotoblas -> xgemm3m_otcopyr -#define XGEMM3M_OTCOPYI gotoblas -> xgemm3m_otcopyi - -#define XGEMM3M_INCOPYB gotoblas -> xgemm3m_incopyb -#define XGEMM3M_INCOPYR gotoblas -> xgemm3m_incopyr -#define XGEMM3M_INCOPYI gotoblas -> xgemm3m_incopyi -#define XGEMM3M_ITCOPYB gotoblas -> xgemm3m_itcopyb -#define XGEMM3M_ITCOPYR gotoblas -> xgemm3m_itcopyr -#define XGEMM3M_ITCOPYI gotoblas -> xgemm3m_itcopyi - -#define XSYMM3M_ILCOPYB gotoblas -> xsymm3m_ilcopyb -#define XSYMM3M_IUCOPYB gotoblas -> xsymm3m_iucopyb -#define XSYMM3M_ILCOPYR gotoblas -> xsymm3m_ilcopyr -#define XSYMM3M_IUCOPYR gotoblas -> xsymm3m_iucopyr -#define XSYMM3M_ILCOPYI gotoblas -> xsymm3m_ilcopyi -#define XSYMM3M_IUCOPYI gotoblas -> xsymm3m_iucopyi - -#define XSYMM3M_OLCOPYB gotoblas -> xsymm3m_olcopyb -#define XSYMM3M_OUCOPYB gotoblas -> xsymm3m_oucopyb -#define XSYMM3M_OLCOPYR gotoblas -> xsymm3m_olcopyr -#define XSYMM3M_OUCOPYR gotoblas -> xsymm3m_oucopyr -#define XSYMM3M_OLCOPYI gotoblas -> xsymm3m_olcopyi -#define XSYMM3M_OUCOPYI gotoblas -> xsymm3m_oucopyi - -#define XHEMM3M_ILCOPYB gotoblas -> xhemm3m_ilcopyb -#define XHEMM3M_IUCOPYB gotoblas -> xhemm3m_iucopyb -#define XHEMM3M_ILCOPYR gotoblas -> xhemm3m_ilcopyr -#define XHEMM3M_IUCOPYR gotoblas -> xhemm3m_iucopyr -#define XHEMM3M_ILCOPYI gotoblas -> xhemm3m_ilcopyi -#define XHEMM3M_IUCOPYI gotoblas -> xhemm3m_iucopyi - -#define XHEMM3M_OLCOPYB gotoblas -> xhemm3m_olcopyb -#define XHEMM3M_OUCOPYB gotoblas -> xhemm3m_oucopyb -#define XHEMM3M_OLCOPYR gotoblas -> xhemm3m_olcopyr -#define XHEMM3M_OUCOPYR gotoblas -> xhemm3m_oucopyr -#define XHEMM3M_OLCOPYI gotoblas -> xhemm3m_olcopyi -#define XHEMM3M_OUCOPYI gotoblas -> xhemm3m_oucopyi - -#define XGEMM3M_KERNEL gotoblas -> xgemm3m_kernel - -#define XNEG_TCOPY gotoblas -> xneg_tcopy -#define XLASWP_NCOPY gotoblas -> xlaswp_ncopy +#define XGEMM_ONCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_oncopy +#define XGEMM_OTCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_otcopy +#define XGEMM_INCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_incopy +#define XGEMM_ITCOPY OPENBLAS_DISPATCH(xgemm) -> xgemm_itcopy + +#define XTRMM_OUNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ounucopy +#define XTRMM_OUTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_outucopy +#define XTRMM_OLNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_olnucopy +#define XTRMM_OLTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_oltucopy +#define XTRSM_OUNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ounucopy +#define XTRSM_OUTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_outucopy +#define XTRSM_OLNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_olnucopy +#define XTRSM_OLTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_oltucopy + +#define XTRMM_IUNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iunucopy +#define XTRMM_IUTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iutucopy +#define XTRMM_ILNUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ilnucopy +#define XTRMM_ILTUCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iltucopy +#define XTRSM_IUNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iunucopy +#define XTRSM_IUTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iutucopy +#define XTRSM_ILNUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ilnucopy +#define XTRSM_ILTUCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iltucopy + +#define XTRMM_OUNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ounncopy +#define XTRMM_OUTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_outncopy +#define XTRMM_OLNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_olnncopy +#define XTRMM_OLTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_oltncopy +#define XTRSM_OUNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ounncopy +#define XTRSM_OUTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_outncopy +#define XTRSM_OLNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_olnncopy +#define XTRSM_OLTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_oltncopy + +#define XTRMM_IUNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iunncopy +#define XTRMM_IUTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iutncopy +#define XTRMM_ILNNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_ilnncopy +#define XTRMM_ILTNCOPY OPENBLAS_DISPATCH(xtrmm) -> xtrmm_iltncopy +#define XTRSM_IUNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iunncopy +#define XTRSM_IUTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iutncopy +#define XTRSM_ILNNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_ilnncopy +#define XTRSM_ILTNCOPY OPENBLAS_DISPATCH(xtrsm) -> xtrsm_iltncopy + +#define XGEMM_BETA OPENBLAS_DISPATCH(xgemm) -> xgemm_beta +#define XGEMM_KERNEL_N OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_n +#define XGEMM_KERNEL_L OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_l +#define XGEMM_KERNEL_R OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_r +#define XGEMM_KERNEL_B OPENBLAS_DISPATCH(xgemm) -> xgemm_kernel_b + +#define XTRMM_KERNEL_LN OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LN +#define XTRMM_KERNEL_LT OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LT +#define XTRMM_KERNEL_LR OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LR +#define XTRMM_KERNEL_LC OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_LC +#define XTRMM_KERNEL_RN OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RN +#define XTRMM_KERNEL_RT OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RT +#define XTRMM_KERNEL_RR OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RR +#define XTRMM_KERNEL_RC OPENBLAS_DISPATCH(xtrmm) -> xtrmm_kernel_RC + +#define XTRSM_KERNEL_LN OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LN +#define XTRSM_KERNEL_LT OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LT +#define XTRSM_KERNEL_LR OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LR +#define XTRSM_KERNEL_LC OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_LC +#define XTRSM_KERNEL_RN OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RN +#define XTRSM_KERNEL_RT OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RT +#define XTRSM_KERNEL_RR OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RR +#define XTRSM_KERNEL_RC OPENBLAS_DISPATCH(xtrsm) -> xtrsm_kernel_RC + +#define XSYMM_IUTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_iutcopy +#define XSYMM_ILTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_iltcopy +#define XSYMM_OUTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_outcopy +#define XSYMM_OLTCOPY OPENBLAS_DISPATCH(xsymm) -> xsymm_oltcopy + +#define XHEMM_OUTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_outcopy +#define XHEMM_OLTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_oltcopy +#define XHEMM_IUTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_iutcopy +#define XHEMM_ILTCOPY OPENBLAS_DISPATCH(xhemm) -> xhemm_iltcopy + +#define XGEMM3M_ONCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyb +#define XGEMM3M_ONCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyr +#define XGEMM3M_ONCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_oncopyi +#define XGEMM3M_OTCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyb +#define XGEMM3M_OTCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyr +#define XGEMM3M_OTCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_otcopyi + +#define XGEMM3M_INCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyb +#define XGEMM3M_INCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyr +#define XGEMM3M_INCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_incopyi +#define XGEMM3M_ITCOPYB OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyb +#define XGEMM3M_ITCOPYR OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyr +#define XGEMM3M_ITCOPYI OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_itcopyi + +#define XSYMM3M_ILCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyb +#define XSYMM3M_IUCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyb +#define XSYMM3M_ILCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyr +#define XSYMM3M_IUCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyr +#define XSYMM3M_ILCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_ilcopyi +#define XSYMM3M_IUCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_iucopyi + +#define XSYMM3M_OLCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyb +#define XSYMM3M_OUCOPYB OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyb +#define XSYMM3M_OLCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyr +#define XSYMM3M_OUCOPYR OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyr +#define XSYMM3M_OLCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_olcopyi +#define XSYMM3M_OUCOPYI OPENBLAS_DISPATCH(xsymm3m) -> xsymm3m_oucopyi + +#define XHEMM3M_ILCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyb +#define XHEMM3M_IUCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyb +#define XHEMM3M_ILCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyr +#define XHEMM3M_IUCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyr +#define XHEMM3M_ILCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_ilcopyi +#define XHEMM3M_IUCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_iucopyi + +#define XHEMM3M_OLCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyb +#define XHEMM3M_OUCOPYB OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyb +#define XHEMM3M_OLCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyr +#define XHEMM3M_OUCOPYR OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyr +#define XHEMM3M_OLCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_olcopyi +#define XHEMM3M_OUCOPYI OPENBLAS_DISPATCH(xhemm3m) -> xhemm3m_oucopyi + +#define XGEMM3M_KERNEL OPENBLAS_DISPATCH(xgemm3m) -> xgemm3m_kernel + +#define XNEG_TCOPY OPENBLAS_DISPATCH(xneg) -> xneg_tcopy +#define XLASWP_NCOPY OPENBLAS_DISPATCH(xlaswp) -> xlaswp_ncopy #endif diff --git a/common_z.h b/common_z.h index abd13dc719..40fb2dd39b 100644 --- a/common_z.h +++ b/common_z.h @@ -290,46 +290,46 @@ #else -#define ZAMAX_K gotoblas -> zamax_k -#define ZAMIN_K gotoblas -> zamin_k +#define ZAMAX_K OPENBLAS_DISPATCH(zamax) -> zamax_k +#define ZAMIN_K OPENBLAS_DISPATCH(zamin) -> zamin_k #define ZMAX_K gotoblas -> zmax_k #define ZMIN_K gotoblas -> zmin_k -#define IZAMAX_K gotoblas -> izamax_k -#define IZAMIN_K gotoblas -> izamin_k +#define IZAMAX_K OPENBLAS_DISPATCH(izamax) -> izamax_k +#define IZAMIN_K OPENBLAS_DISPATCH(izamin) -> izamin_k #define IZMAX_K gotoblas -> izmax_k #define IZMIN_K gotoblas -> izmin_k -#define ZASUM_K gotoblas -> zasum_k -#define ZAXPYU_K gotoblas -> zaxpy_k -#define ZAXPYC_K gotoblas -> zaxpyc_k -#define ZCOPY_K gotoblas -> zcopy_k -#define ZDOTU_K gotoblas -> zdotu_k -#define ZDOTC_K gotoblas -> zdotc_k -#define ZNRM2_K gotoblas -> znrm2_k -#define ZSCAL_K gotoblas -> zscal_k -#define ZSUM_K gotoblas -> zsum_k -#define ZSWAP_K gotoblas -> zswap_k -#define ZROT_K gotoblas -> zdrot_k - -#define ZGEMV_N gotoblas -> zgemv_n -#define ZGEMV_T gotoblas -> zgemv_t -#define ZGEMV_R gotoblas -> zgemv_r -#define ZGEMV_C gotoblas -> zgemv_c -#define ZGEMV_O gotoblas -> zgemv_o -#define ZGEMV_U gotoblas -> zgemv_u -#define ZGEMV_S gotoblas -> zgemv_s -#define ZGEMV_D gotoblas -> zgemv_d - -#define ZGERU_K gotoblas -> zgeru_k -#define ZGERC_K gotoblas -> zgerc_k -#define ZGERV_K gotoblas -> zgerv_k -#define ZGERD_K gotoblas -> zgerd_k - -#define ZSYMV_U gotoblas -> zsymv_U -#define ZSYMV_L gotoblas -> zsymv_L -#define ZHEMV_U gotoblas -> zhemv_U -#define ZHEMV_L gotoblas -> zhemv_L -#define ZHEMV_V gotoblas -> zhemv_V -#define ZHEMV_M gotoblas -> zhemv_M +#define ZASUM_K OPENBLAS_DISPATCH(zasum) -> zasum_k +#define ZAXPYU_K OPENBLAS_DISPATCH(zaxpy) -> zaxpy_k +#define ZAXPYC_K OPENBLAS_DISPATCH(zaxpyc) -> zaxpyc_k +#define ZCOPY_K OPENBLAS_DISPATCH(zcopy) -> zcopy_k +#define ZDOTU_K OPENBLAS_DISPATCH(zdotu) -> zdotu_k +#define ZDOTC_K OPENBLAS_DISPATCH(zdotc) -> zdotc_k +#define ZNRM2_K OPENBLAS_DISPATCH(znrm2) -> znrm2_k +#define ZSCAL_K OPENBLAS_DISPATCH(zscal) -> zscal_k +#define ZSUM_K OPENBLAS_DISPATCH(zsum) -> zsum_k +#define ZSWAP_K OPENBLAS_DISPATCH(zswap) -> zswap_k +#define ZROT_K OPENBLAS_DISPATCH(zdrot) -> zdrot_k + +#define ZGEMV_N OPENBLAS_DISPATCH(zgemv) -> zgemv_n +#define ZGEMV_T OPENBLAS_DISPATCH(zgemv) -> zgemv_t +#define ZGEMV_R OPENBLAS_DISPATCH(zgemv) -> zgemv_r +#define ZGEMV_C OPENBLAS_DISPATCH(zgemv) -> zgemv_c +#define ZGEMV_O OPENBLAS_DISPATCH(zgemv) -> zgemv_o +#define ZGEMV_U OPENBLAS_DISPATCH(zgemv) -> zgemv_u +#define ZGEMV_S OPENBLAS_DISPATCH(zgemv) -> zgemv_s +#define ZGEMV_D OPENBLAS_DISPATCH(zgemv) -> zgemv_d + +#define ZGERU_K OPENBLAS_DISPATCH(zgeru) -> zgeru_k +#define ZGERC_K OPENBLAS_DISPATCH(zgerc) -> zgerc_k +#define ZGERV_K OPENBLAS_DISPATCH(zgerv) -> zgerv_k +#define ZGERD_K OPENBLAS_DISPATCH(zgerd) -> zgerd_k + +#define ZSYMV_U OPENBLAS_DISPATCH(zsymv) -> zsymv_U +#define ZSYMV_L OPENBLAS_DISPATCH(zsymv) -> zsymv_L +#define ZHEMV_U OPENBLAS_DISPATCH(zhemv) -> zhemv_U +#define ZHEMV_L OPENBLAS_DISPATCH(zhemv) -> zhemv_L +#define ZHEMV_V OPENBLAS_DISPATCH(zhemv) -> zhemv_V +#define ZHEMV_M OPENBLAS_DISPATCH(zhemv) -> zhemv_M #define ZSYMV_THREAD_U zsymv_thread_U #define ZSYMV_THREAD_L zsymv_thread_L @@ -338,209 +338,209 @@ #define ZHEMV_THREAD_V zhemv_thread_V #define ZHEMV_THREAD_M zhemv_thread_M -#define ZGEMM_ONCOPY gotoblas -> zgemm_oncopy -#define ZGEMM_OTCOPY gotoblas -> zgemm_otcopy -#define ZGEMM_INCOPY gotoblas -> zgemm_incopy -#define ZGEMM_ITCOPY gotoblas -> zgemm_itcopy - -#define ZTRMM_OUNUCOPY gotoblas -> ztrmm_ounucopy -#define ZTRMM_OUTUCOPY gotoblas -> ztrmm_outucopy -#define ZTRMM_OLNUCOPY gotoblas -> ztrmm_olnucopy -#define ZTRMM_OLTUCOPY gotoblas -> ztrmm_oltucopy -#define ZTRSM_OUNUCOPY gotoblas -> ztrsm_ounucopy -#define ZTRSM_OUTUCOPY gotoblas -> ztrsm_outucopy -#define ZTRSM_OLNUCOPY gotoblas -> ztrsm_olnucopy -#define ZTRSM_OLTUCOPY gotoblas -> ztrsm_oltucopy - -#define ZTRMM_IUNUCOPY gotoblas -> ztrmm_iunucopy -#define ZTRMM_IUTUCOPY gotoblas -> ztrmm_iutucopy -#define ZTRMM_ILNUCOPY gotoblas -> ztrmm_ilnucopy -#define ZTRMM_ILTUCOPY gotoblas -> ztrmm_iltucopy -#define ZTRSM_IUNUCOPY gotoblas -> ztrsm_iunucopy -#define ZTRSM_IUTUCOPY gotoblas -> ztrsm_iutucopy -#define ZTRSM_ILNUCOPY gotoblas -> ztrsm_ilnucopy -#define ZTRSM_ILTUCOPY gotoblas -> ztrsm_iltucopy - -#define ZTRMM_OUNNCOPY gotoblas -> ztrmm_ounncopy -#define ZTRMM_OUTNCOPY gotoblas -> ztrmm_outncopy -#define ZTRMM_OLNNCOPY gotoblas -> ztrmm_olnncopy -#define ZTRMM_OLTNCOPY gotoblas -> ztrmm_oltncopy -#define ZTRSM_OUNNCOPY gotoblas -> ztrsm_ounncopy -#define ZTRSM_OUTNCOPY gotoblas -> ztrsm_outncopy -#define ZTRSM_OLNNCOPY gotoblas -> ztrsm_olnncopy -#define ZTRSM_OLTNCOPY gotoblas -> ztrsm_oltncopy - -#define ZTRMM_IUNNCOPY gotoblas -> ztrmm_iunncopy -#define ZTRMM_IUTNCOPY gotoblas -> ztrmm_iutncopy -#define ZTRMM_ILNNCOPY gotoblas -> ztrmm_ilnncopy -#define ZTRMM_ILTNCOPY gotoblas -> ztrmm_iltncopy -#define ZSYMM_INCOPY gotoblas -> zsymm_incopy -#define ZSYMM_ITCOPY gotoblas -> zsymm_itcopy -#define ZTRMM_INCOPY gotoblas -> ztrmm_incopy -#define ZTRMM_ITCOPY gotoblas -> ztrmm_itcopy - -#define ZTRSM_IUNNCOPY gotoblas -> ztrsm_iunncopy -#define ZTRSM_IUTNCOPY gotoblas -> ztrsm_iutncopy -#define ZTRSM_ILNNCOPY gotoblas -> ztrsm_ilnncopy -#define ZTRSM_ILTNCOPY gotoblas -> ztrsm_iltncopy - -#define ZGEMM_BETA gotoblas -> zgemm_beta -#define SME_ZGEMM_KERNEL gotoblas -> sme_zgemm_kernel -#define ZGEMM_KERNEL_N gotoblas -> zgemm_kernel_n -#define ZGEMM_KERNEL_L gotoblas -> zgemm_kernel_l -#define ZGEMM_KERNEL_R gotoblas -> zgemm_kernel_r -#define ZGEMM_KERNEL_B gotoblas -> zgemm_kernel_b - -#define ZSYMM_KERNEL_N gotoblas -> zsymm_kernel_n -#define ZSYMM_KERNEL_L gotoblas -> zsymm_kernel_l -#define ZSYMM_KERNEL_R gotoblas -> zsymm_kernel_r -#define ZSYMM_KERNEL_B gotoblas -> zsymm_kernel_b -#define ZTRMM_GEMM_KERNEL_N gotoblas -> ztrmm_gemm_kernel_n -#define ZTRMM_GEMM_KERNEL_L gotoblas -> ztrmm_gemm_kernel_l -#define ZTRMM_GEMM_KERNEL_R gotoblas -> ztrmm_gemm_kernel_r -#define ZTRMM_GEMM_KERNEL_B gotoblas -> ztrmm_gemm_kernel_b - -#define ZTRMM_KERNEL_LN gotoblas -> ztrmm_kernel_LN -#define ZTRMM_KERNEL_LT gotoblas -> ztrmm_kernel_LT -#define ZTRMM_KERNEL_LR gotoblas -> ztrmm_kernel_LR -#define ZTRMM_KERNEL_LC gotoblas -> ztrmm_kernel_LC -#define ZTRMM_KERNEL_RN gotoblas -> ztrmm_kernel_RN -#define ZTRMM_KERNEL_RT gotoblas -> ztrmm_kernel_RT -#define ZTRMM_KERNEL_RR gotoblas -> ztrmm_kernel_RR -#define ZTRMM_KERNEL_RC gotoblas -> ztrmm_kernel_RC - -#define ZTRSM_KERNEL_LN gotoblas -> ztrsm_kernel_LN -#define ZTRSM_KERNEL_LT gotoblas -> ztrsm_kernel_LT -#define ZTRSM_KERNEL_LR gotoblas -> ztrsm_kernel_LR -#define ZTRSM_KERNEL_LC gotoblas -> ztrsm_kernel_LC -#define ZTRSM_KERNEL_RN gotoblas -> ztrsm_kernel_RN -#define ZTRSM_KERNEL_RT gotoblas -> ztrsm_kernel_RT -#define ZTRSM_KERNEL_RR gotoblas -> ztrsm_kernel_RR -#define ZTRSM_KERNEL_RC gotoblas -> ztrsm_kernel_RC - -#define ZSYMM_IUTCOPY gotoblas -> zsymm_iutcopy -#define ZSYMM_ILTCOPY gotoblas -> zsymm_iltcopy -#define ZSYMM_OUTCOPY gotoblas -> zsymm_outcopy -#define ZSYMM_OLTCOPY gotoblas -> zsymm_oltcopy - -#define ZHEMM_OUTCOPY gotoblas -> zhemm_outcopy -#define ZHEMM_OLTCOPY gotoblas -> zhemm_oltcopy -#define ZHEMM_IUTCOPY gotoblas -> zhemm_iutcopy -#define ZHEMM_ILTCOPY gotoblas -> zhemm_iltcopy - -#define ZGEMM3M_ONCOPYB gotoblas -> zgemm3m_oncopyb -#define ZGEMM3M_ONCOPYR gotoblas -> zgemm3m_oncopyr -#define ZGEMM3M_ONCOPYI gotoblas -> zgemm3m_oncopyi -#define ZGEMM3M_OTCOPYB gotoblas -> zgemm3m_otcopyb -#define ZGEMM3M_OTCOPYR gotoblas -> zgemm3m_otcopyr -#define ZGEMM3M_OTCOPYI gotoblas -> zgemm3m_otcopyi - -#define ZGEMM3M_INCOPYB gotoblas -> zgemm3m_incopyb -#define ZGEMM3M_INCOPYR gotoblas -> zgemm3m_incopyr -#define ZGEMM3M_INCOPYI gotoblas -> zgemm3m_incopyi -#define ZGEMM3M_ITCOPYB gotoblas -> zgemm3m_itcopyb -#define ZGEMM3M_ITCOPYR gotoblas -> zgemm3m_itcopyr -#define ZGEMM3M_ITCOPYI gotoblas -> zgemm3m_itcopyi - -#define ZSYMM3M_ILCOPYB gotoblas -> zsymm3m_ilcopyb -#define ZSYMM3M_IUCOPYB gotoblas -> zsymm3m_iucopyb -#define ZSYMM3M_ILCOPYR gotoblas -> zsymm3m_ilcopyr -#define ZSYMM3M_IUCOPYR gotoblas -> zsymm3m_iucopyr -#define ZSYMM3M_ILCOPYI gotoblas -> zsymm3m_ilcopyi -#define ZSYMM3M_IUCOPYI gotoblas -> zsymm3m_iucopyi - -#define ZSYMM3M_OLCOPYB gotoblas -> zsymm3m_olcopyb -#define ZSYMM3M_OUCOPYB gotoblas -> zsymm3m_oucopyb -#define ZSYMM3M_OLCOPYR gotoblas -> zsymm3m_olcopyr -#define ZSYMM3M_OUCOPYR gotoblas -> zsymm3m_oucopyr -#define ZSYMM3M_OLCOPYI gotoblas -> zsymm3m_olcopyi -#define ZSYMM3M_OUCOPYI gotoblas -> zsymm3m_oucopyi - -#define ZHEMM3M_ILCOPYB gotoblas -> zhemm3m_ilcopyb -#define ZHEMM3M_IUCOPYB gotoblas -> zhemm3m_iucopyb -#define ZHEMM3M_ILCOPYR gotoblas -> zhemm3m_ilcopyr -#define ZHEMM3M_IUCOPYR gotoblas -> zhemm3m_iucopyr -#define ZHEMM3M_ILCOPYI gotoblas -> zhemm3m_ilcopyi -#define ZHEMM3M_IUCOPYI gotoblas -> zhemm3m_iucopyi - -#define ZHEMM3M_OLCOPYB gotoblas -> zhemm3m_olcopyb -#define ZHEMM3M_OUCOPYB gotoblas -> zhemm3m_oucopyb -#define ZHEMM3M_OLCOPYR gotoblas -> zhemm3m_olcopyr -#define ZHEMM3M_OUCOPYR gotoblas -> zhemm3m_oucopyr -#define ZHEMM3M_OLCOPYI gotoblas -> zhemm3m_olcopyi -#define ZHEMM3M_OUCOPYI gotoblas -> zhemm3m_oucopyi - -#define ZGEMM3M_KERNEL gotoblas -> zgemm3m_kernel - -#define ZNEG_TCOPY gotoblas -> zneg_tcopy -#define ZLASWP_NCOPY gotoblas -> zlaswp_ncopy - -#define ZAXPBY_K gotoblas -> zaxpby_k - -#define ZOMATCOPY_K_CN gotoblas -> zomatcopy_k_cn -#define ZOMATCOPY_K_RN gotoblas -> zomatcopy_k_rn -#define ZOMATCOPY_K_CT gotoblas -> zomatcopy_k_ct -#define ZOMATCOPY_K_RT gotoblas -> zomatcopy_k_rt -#define ZOMATCOPY_K_CNC gotoblas -> zomatcopy_k_cnc -#define ZOMATCOPY_K_RNC gotoblas -> zomatcopy_k_rnc -#define ZOMATCOPY_K_CTC gotoblas -> zomatcopy_k_ctc -#define ZOMATCOPY_K_RTC gotoblas -> zomatcopy_k_rtc - -#define ZIMATCOPY_K_CN gotoblas -> zimatcopy_k_cn -#define ZIMATCOPY_K_RN gotoblas -> zimatcopy_k_rn -#define ZIMATCOPY_K_CT gotoblas -> zimatcopy_k_ct -#define ZIMATCOPY_K_RT gotoblas -> zimatcopy_k_rt -#define ZIMATCOPY_K_CNC gotoblas -> zimatcopy_k_cnc -#define ZIMATCOPY_K_RNC gotoblas -> zimatcopy_k_rnc -#define ZIMATCOPY_K_CTC gotoblas -> zimatcopy_k_ctc -#define ZIMATCOPY_K_RTC gotoblas -> zimatcopy_k_rtc - -#define ZGEADD_K gotoblas -> zgeadd_k - -#define ZGEMM_SMALL_MATRIX_PERMIT gotoblas -> zgemm_small_matrix_permit +#define ZGEMM_ONCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_oncopy +#define ZGEMM_OTCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_otcopy +#define ZGEMM_INCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_incopy +#define ZGEMM_ITCOPY OPENBLAS_DISPATCH(zgemm) -> zgemm_itcopy + +#define ZTRMM_OUNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ounucopy +#define ZTRMM_OUTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_outucopy +#define ZTRMM_OLNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_olnucopy +#define ZTRMM_OLTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_oltucopy +#define ZTRSM_OUNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ounucopy +#define ZTRSM_OUTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_outucopy +#define ZTRSM_OLNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_olnucopy +#define ZTRSM_OLTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_oltucopy + +#define ZTRMM_IUNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iunucopy +#define ZTRMM_IUTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iutucopy +#define ZTRMM_ILNUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ilnucopy +#define ZTRMM_ILTUCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iltucopy +#define ZTRSM_IUNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iunucopy +#define ZTRSM_IUTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iutucopy +#define ZTRSM_ILNUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ilnucopy +#define ZTRSM_ILTUCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iltucopy + +#define ZTRMM_OUNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ounncopy +#define ZTRMM_OUTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_outncopy +#define ZTRMM_OLNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_olnncopy +#define ZTRMM_OLTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_oltncopy +#define ZTRSM_OUNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ounncopy +#define ZTRSM_OUTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_outncopy +#define ZTRSM_OLNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_olnncopy +#define ZTRSM_OLTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_oltncopy + +#define ZTRMM_IUNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iunncopy +#define ZTRMM_IUTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iutncopy +#define ZTRMM_ILNNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_ilnncopy +#define ZTRMM_ILTNCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_iltncopy +#define ZSYMM_INCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_incopy +#define ZSYMM_ITCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_itcopy +#define ZTRMM_INCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_incopy +#define ZTRMM_ITCOPY OPENBLAS_DISPATCH(ztrmm) -> ztrmm_itcopy + +#define ZTRSM_IUNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iunncopy +#define ZTRSM_IUTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iutncopy +#define ZTRSM_ILNNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_ilnncopy +#define ZTRSM_ILTNCOPY OPENBLAS_DISPATCH(ztrsm) -> ztrsm_iltncopy + +#define ZGEMM_BETA OPENBLAS_DISPATCH(zgemm) -> zgemm_beta +#define SME_ZGEMM_KERNEL OPENBLAS_DISPATCH(zgemm) -> sme_zgemm_kernel +#define ZGEMM_KERNEL_N OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_n +#define ZGEMM_KERNEL_L OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_l +#define ZGEMM_KERNEL_R OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_r +#define ZGEMM_KERNEL_B OPENBLAS_DISPATCH(zgemm) -> zgemm_kernel_b + +#define ZSYMM_KERNEL_N OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_n +#define ZSYMM_KERNEL_L OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_l +#define ZSYMM_KERNEL_R OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_r +#define ZSYMM_KERNEL_B OPENBLAS_DISPATCH(zsymm) -> zsymm_kernel_b +#define ZTRMM_GEMM_KERNEL_N OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_n +#define ZTRMM_GEMM_KERNEL_L OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_l +#define ZTRMM_GEMM_KERNEL_R OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_r +#define ZTRMM_GEMM_KERNEL_B OPENBLAS_DISPATCH(ztrmm) -> ztrmm_gemm_kernel_b + +#define ZTRMM_KERNEL_LN OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LN +#define ZTRMM_KERNEL_LT OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LT +#define ZTRMM_KERNEL_LR OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LR +#define ZTRMM_KERNEL_LC OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_LC +#define ZTRMM_KERNEL_RN OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RN +#define ZTRMM_KERNEL_RT OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RT +#define ZTRMM_KERNEL_RR OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RR +#define ZTRMM_KERNEL_RC OPENBLAS_DISPATCH(ztrmm) -> ztrmm_kernel_RC + +#define ZTRSM_KERNEL_LN OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LN +#define ZTRSM_KERNEL_LT OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LT +#define ZTRSM_KERNEL_LR OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LR +#define ZTRSM_KERNEL_LC OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_LC +#define ZTRSM_KERNEL_RN OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RN +#define ZTRSM_KERNEL_RT OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RT +#define ZTRSM_KERNEL_RR OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RR +#define ZTRSM_KERNEL_RC OPENBLAS_DISPATCH(ztrsm) -> ztrsm_kernel_RC + +#define ZSYMM_IUTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_iutcopy +#define ZSYMM_ILTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_iltcopy +#define ZSYMM_OUTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_outcopy +#define ZSYMM_OLTCOPY OPENBLAS_DISPATCH(zsymm) -> zsymm_oltcopy + +#define ZHEMM_OUTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_outcopy +#define ZHEMM_OLTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_oltcopy +#define ZHEMM_IUTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_iutcopy +#define ZHEMM_ILTCOPY OPENBLAS_DISPATCH(zhemm) -> zhemm_iltcopy + +#define ZGEMM3M_ONCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyb +#define ZGEMM3M_ONCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyr +#define ZGEMM3M_ONCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_oncopyi +#define ZGEMM3M_OTCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyb +#define ZGEMM3M_OTCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyr +#define ZGEMM3M_OTCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_otcopyi + +#define ZGEMM3M_INCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyb +#define ZGEMM3M_INCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyr +#define ZGEMM3M_INCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_incopyi +#define ZGEMM3M_ITCOPYB OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyb +#define ZGEMM3M_ITCOPYR OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyr +#define ZGEMM3M_ITCOPYI OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_itcopyi + +#define ZSYMM3M_ILCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyb +#define ZSYMM3M_IUCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyb +#define ZSYMM3M_ILCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyr +#define ZSYMM3M_IUCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyr +#define ZSYMM3M_ILCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_ilcopyi +#define ZSYMM3M_IUCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_iucopyi + +#define ZSYMM3M_OLCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyb +#define ZSYMM3M_OUCOPYB OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyb +#define ZSYMM3M_OLCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyr +#define ZSYMM3M_OUCOPYR OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyr +#define ZSYMM3M_OLCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_olcopyi +#define ZSYMM3M_OUCOPYI OPENBLAS_DISPATCH(zsymm3m) -> zsymm3m_oucopyi + +#define ZHEMM3M_ILCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyb +#define ZHEMM3M_IUCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyb +#define ZHEMM3M_ILCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyr +#define ZHEMM3M_IUCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyr +#define ZHEMM3M_ILCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_ilcopyi +#define ZHEMM3M_IUCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_iucopyi + +#define ZHEMM3M_OLCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyb +#define ZHEMM3M_OUCOPYB OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyb +#define ZHEMM3M_OLCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyr +#define ZHEMM3M_OUCOPYR OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyr +#define ZHEMM3M_OLCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_olcopyi +#define ZHEMM3M_OUCOPYI OPENBLAS_DISPATCH(zhemm3m) -> zhemm3m_oucopyi + +#define ZGEMM3M_KERNEL OPENBLAS_DISPATCH(zgemm3m) -> zgemm3m_kernel + +#define ZNEG_TCOPY OPENBLAS_DISPATCH(zneg) -> zneg_tcopy +#define ZLASWP_NCOPY OPENBLAS_DISPATCH(zlaswp) -> zlaswp_ncopy + +#define ZAXPBY_K OPENBLAS_DISPATCH(zaxpby) -> zaxpby_k + +#define ZOMATCOPY_K_CN OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_cn +#define ZOMATCOPY_K_RN OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rn +#define ZOMATCOPY_K_CT OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_ct +#define ZOMATCOPY_K_RT OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rt +#define ZOMATCOPY_K_CNC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_cnc +#define ZOMATCOPY_K_RNC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rnc +#define ZOMATCOPY_K_CTC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_ctc +#define ZOMATCOPY_K_RTC OPENBLAS_DISPATCH(zomatcopy) -> zomatcopy_k_rtc + +#define ZIMATCOPY_K_CN OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_cn +#define ZIMATCOPY_K_RN OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rn +#define ZIMATCOPY_K_CT OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_ct +#define ZIMATCOPY_K_RT OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rt +#define ZIMATCOPY_K_CNC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_cnc +#define ZIMATCOPY_K_RNC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rnc +#define ZIMATCOPY_K_CTC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_ctc +#define ZIMATCOPY_K_RTC OPENBLAS_DISPATCH(zimatcopy) -> zimatcopy_k_rtc + +#define ZGEADD_K OPENBLAS_DISPATCH(zgeadd) -> zgeadd_k + +#define ZGEMM_SMALL_MATRIX_PERMIT OPENBLAS_DISPATCH(zgemm) -> zgemm_small_matrix_permit #endif -#define ZGEMM_SMALL_KERNEL_BASE FUNC_BASE(zgemm_small_kernel_nn) -#define ZGEMM_SMALL_KERNEL_NN FUNC_OFFSET(zgemm_small_kernel_nn) -#define ZGEMM_SMALL_KERNEL_NT FUNC_OFFSET(zgemm_small_kernel_nt) -#define ZGEMM_SMALL_KERNEL_NR FUNC_OFFSET(zgemm_small_kernel_nr) -#define ZGEMM_SMALL_KERNEL_NC FUNC_OFFSET(zgemm_small_kernel_nc) - -#define ZGEMM_SMALL_KERNEL_TN FUNC_OFFSET(zgemm_small_kernel_tn) -#define ZGEMM_SMALL_KERNEL_TT FUNC_OFFSET(zgemm_small_kernel_tt) -#define ZGEMM_SMALL_KERNEL_TR FUNC_OFFSET(zgemm_small_kernel_tr) -#define ZGEMM_SMALL_KERNEL_TC FUNC_OFFSET(zgemm_small_kernel_tc) - -#define ZGEMM_SMALL_KERNEL_RN FUNC_OFFSET(zgemm_small_kernel_rn) -#define ZGEMM_SMALL_KERNEL_RT FUNC_OFFSET(zgemm_small_kernel_rt) -#define ZGEMM_SMALL_KERNEL_RR FUNC_OFFSET(zgemm_small_kernel_rr) -#define ZGEMM_SMALL_KERNEL_RC FUNC_OFFSET(zgemm_small_kernel_rc) - -#define ZGEMM_SMALL_KERNEL_CN FUNC_OFFSET(zgemm_small_kernel_cn) -#define ZGEMM_SMALL_KERNEL_CT FUNC_OFFSET(zgemm_small_kernel_ct) -#define ZGEMM_SMALL_KERNEL_CR FUNC_OFFSET(zgemm_small_kernel_cr) -#define ZGEMM_SMALL_KERNEL_CC FUNC_OFFSET(zgemm_small_kernel_cc) - -#define ZGEMM_SMALL_KERNEL_B0_NN FUNC_OFFSET(zgemm_small_kernel_b0_nn) -#define ZGEMM_SMALL_KERNEL_B0_NT FUNC_OFFSET(zgemm_small_kernel_b0_nt) -#define ZGEMM_SMALL_KERNEL_B0_NR FUNC_OFFSET(zgemm_small_kernel_b0_nr) -#define ZGEMM_SMALL_KERNEL_B0_NC FUNC_OFFSET(zgemm_small_kernel_b0_nc) - -#define ZGEMM_SMALL_KERNEL_B0_TN FUNC_OFFSET(zgemm_small_kernel_b0_tn) -#define ZGEMM_SMALL_KERNEL_B0_TT FUNC_OFFSET(zgemm_small_kernel_b0_tt) -#define ZGEMM_SMALL_KERNEL_B0_TR FUNC_OFFSET(zgemm_small_kernel_b0_tr) -#define ZGEMM_SMALL_KERNEL_B0_TC FUNC_OFFSET(zgemm_small_kernel_b0_tc) - -#define ZGEMM_SMALL_KERNEL_B0_RN FUNC_OFFSET(zgemm_small_kernel_b0_rn) -#define ZGEMM_SMALL_KERNEL_B0_RT FUNC_OFFSET(zgemm_small_kernel_b0_rt) -#define ZGEMM_SMALL_KERNEL_B0_RR FUNC_OFFSET(zgemm_small_kernel_b0_rr) -#define ZGEMM_SMALL_KERNEL_B0_RC FUNC_OFFSET(zgemm_small_kernel_b0_rc) - -#define ZGEMM_SMALL_KERNEL_B0_CN FUNC_OFFSET(zgemm_small_kernel_b0_cn) -#define ZGEMM_SMALL_KERNEL_B0_CT FUNC_OFFSET(zgemm_small_kernel_b0_ct) -#define ZGEMM_SMALL_KERNEL_B0_CR FUNC_OFFSET(zgemm_small_kernel_b0_cr) -#define ZGEMM_SMALL_KERNEL_B0_CC FUNC_OFFSET(zgemm_small_kernel_b0_cc) +#define ZGEMM_SMALL_KERNEL_BASE OPENBLAS_DISPATCH_BASE(zgemm) +#define ZGEMM_SMALL_KERNEL_NN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nn) +#define ZGEMM_SMALL_KERNEL_NT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nt) +#define ZGEMM_SMALL_KERNEL_NR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nr) +#define ZGEMM_SMALL_KERNEL_NC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_nc) + +#define ZGEMM_SMALL_KERNEL_TN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tn) +#define ZGEMM_SMALL_KERNEL_TT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tt) +#define ZGEMM_SMALL_KERNEL_TR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tr) +#define ZGEMM_SMALL_KERNEL_TC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_tc) + +#define ZGEMM_SMALL_KERNEL_RN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rn) +#define ZGEMM_SMALL_KERNEL_RT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rt) +#define ZGEMM_SMALL_KERNEL_RR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rr) +#define ZGEMM_SMALL_KERNEL_RC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_rc) + +#define ZGEMM_SMALL_KERNEL_CN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cn) +#define ZGEMM_SMALL_KERNEL_CT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_ct) +#define ZGEMM_SMALL_KERNEL_CR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cr) +#define ZGEMM_SMALL_KERNEL_CC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_cc) + +#define ZGEMM_SMALL_KERNEL_B0_NN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nn) +#define ZGEMM_SMALL_KERNEL_B0_NT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nt) +#define ZGEMM_SMALL_KERNEL_B0_NR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nr) +#define ZGEMM_SMALL_KERNEL_B0_NC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_nc) + +#define ZGEMM_SMALL_KERNEL_B0_TN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tn) +#define ZGEMM_SMALL_KERNEL_B0_TT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tt) +#define ZGEMM_SMALL_KERNEL_B0_TR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tr) +#define ZGEMM_SMALL_KERNEL_B0_TC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_tc) + +#define ZGEMM_SMALL_KERNEL_B0_RN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rn) +#define ZGEMM_SMALL_KERNEL_B0_RT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rt) +#define ZGEMM_SMALL_KERNEL_B0_RR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rr) +#define ZGEMM_SMALL_KERNEL_B0_RC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_rc) + +#define ZGEMM_SMALL_KERNEL_B0_CN OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cn) +#define ZGEMM_SMALL_KERNEL_B0_CT OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_ct) +#define ZGEMM_SMALL_KERNEL_B0_CR OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cr) +#define ZGEMM_SMALL_KERNEL_B0_CC OPENBLAS_DISPATCH_OFFSET(zgemm, zgemm_small_kernel_b0_cc) #define ZGEMM_NN zgemm_nn diff --git a/driver/others/dispatch.c b/driver/others/dispatch.c index cecde58205..f2241298af 100644 --- a/driver/others/dispatch.c +++ b/driver/others/dispatch.c @@ -47,3 +47,939 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. const openblas_##group##_dispatch_t *const openblas_##group##_dispatch[OPENBLAS_NUM_CORES] = { \ OPENBLAS_CORE_LIST(DISPATCH_ENTRY, group) \ }; + +#if BUILD_HFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(shgemm) +#endif + +#if BUILD_HFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(shgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbstobf16) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbdtobf16) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbf16tos) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(dbf16tod) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbamax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbamin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbmax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbmin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbamax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbamin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbmax) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(isbmin) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbnrm2) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbasum) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsum) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbcopy) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbdot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(dsbdot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbrot) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbrotm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bscal) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbaxpy) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbscal) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbswap) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbgemv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbger) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsymv) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(bgemm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbgemm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbtrsm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbtrmm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbsymm) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sbneg) +#endif + +#if BUILD_BFLOAT16 == 1 +DEFINE_DISPATCH_TABLE(sblaswp) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(samax) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(samin) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(smax) +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +DEFINE_DISPATCH_TABLE(smin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(isamax) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(isamin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ismax) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ismin) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(snrm2) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sasum) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(ssum) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(scopy) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sdot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(srot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(srotm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(saxpy) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(sscal) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sswap) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sgemv) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sger) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(ssymv) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(sgemm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ssymm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(strmm) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +DEFINE_DISPATCH_TABLE(ssyrk) +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +DEFINE_DISPATCH_TABLE(ssyr2k) +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(strsm) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sneg) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(slaswp) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(damax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(damin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dmax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dmin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idamax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idamin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idmax) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(idmin) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dnrm2) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dasum) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsum) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dcopy) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(ddot) +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsdot) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(drot) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(drotm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(daxpy) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dscal) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dswap) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dgemv) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dger) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dsymv) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dgemm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dsymm) +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(dtrmm) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dtrsm) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dneg) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dlaswp) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qmax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qmin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqmax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(iqmin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qnrm2) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qasum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qcopy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qdot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qrot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qrotm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qaxpy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qscal) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qswap) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qgemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qger) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsymv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qgemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qtrsm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qtrmm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qsymm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qneg) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(qlaswp) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(camax) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(camin) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(icamax) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(icamin) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cnrm2) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(casum) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csum) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ccopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cdotu) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cdotc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csrot) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpyc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cscal) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cswap) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgeru) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerc) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgerd) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemv) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ctrmm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(ctrsm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemm) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgemm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(csymm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(chemm3m) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cneg) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(claswp) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zamax) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zamin) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(izamax) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(izamin) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(znrm2) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zasum) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsum) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zcopy) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdotu) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdotc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zdrot) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zaxpy) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zaxpyc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zscal) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zswap) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgeru) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerc) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgerd) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemv) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(ztrmm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(ztrsm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemm) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zgemm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zsymm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zhemm3m) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zneg) +#endif + +#if (BUILD_COMPLEX16 == 1) +DEFINE_DISPATCH_TABLE(zlaswp) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(ixamax) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(ixamin) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xnrm2) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xasum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsum) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xcopy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xdotu) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xdotc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xqrot) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xaxpy) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xaxpyc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xscal) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xswap) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgeru) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerc) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgerd) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemv) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xtrsm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xtrmm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemm) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xgemm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xsymm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xhemm3m) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xneg) +#endif + +#ifdef EXPRECISION +DEFINE_DISPATCH_TABLE(xlaswp) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(saxpby) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(daxpby) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(caxpby) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zaxpby) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(somatcopy) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(domatcopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(comatcopy) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zomatcopy) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(simatcopy) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dimatcopy) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cimatcopy) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zimatcopy) +#endif + +#if (BUILD_SINGLE==1) +DEFINE_DISPATCH_TABLE(sgeadd) +#endif + +#if (BUILD_DOUBLE==1) +DEFINE_DISPATCH_TABLE(dgeadd) +#endif + +#if (BUILD_COMPLEX==1) +DEFINE_DISPATCH_TABLE(cgeadd) +#endif + +#if (BUILD_COMPLEX16==1) +DEFINE_DISPATCH_TABLE(zgeadd) +#endif diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 65a17733dc..5478ab0c61 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -52,21 +52,14 @@ static void init_parameter(void); gotoblas_t TABLE_NAME = { .core = OPENBLAS_CORETS, - .dtb_entries = DTB_DEFAULT_ENTRIES, - .switch_ratio = SWITCH_RATIO, - .divide_rate = GEMM_DIVIDE_RATE, - .divide_limit = GEMM_DIVIDE_LIMIT, - .preferred_size = GEMM_PREFERRED_SIZE, - .offsetA = GEMM_DEFAULT_OFFSET_A, .offsetB = GEMM_DEFAULT_OFFSET_B, .align = GEMM_DEFAULT_ALIGN, - #ifdef BUILD_HFLOAT16 .shgemm_p = 0, .shgemm_q = 0, @@ -78,21 +71,7 @@ gotoblas_t TABLE_NAME = { #else .shgemm_unroll_mn = MAX(SHGEMM_DEFAULT_UNROLL_M, SHGEMM_DEFAULT_UNROLL_N), #endif - .shgemm_kernel = shgemm_kernelTS, - .shgemm_beta = shgemm_betaTS, -#if SHGEMM_DEFAULT_UNROLL_M != SHGEMM_DEFAULT_UNROLL_N - .shgemm_incopy = shgemm_incopyTS, - .shgemm_itcopy = shgemm_itcopyTS, -#else - .shgemm_incopy = shgemm_oncopyTS, - .shgemm_itcopy = shgemm_otcopyTS, #endif - .shgemm_oncopy = shgemm_oncopyTS, - .shgemm_otcopy = shgemm_otcopyTS, - .shgemv_n = shgemv_nTS, - .shgemv_t = shgemv_tTS, -#endif - #ifdef BUILD_BFLOAT16 .bgemm_p = 0, .bgemm_q = 0, @@ -105,7 +84,6 @@ gotoblas_t TABLE_NAME = { .bgemm_unroll_mn = MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N), #endif .bgemm_align_k = BGEMM_ALIGN_K, - .sbgemm_p = 0, .sbgemm_q = 0, .sbgemm_r = 0, @@ -116,43 +94,330 @@ gotoblas_t TABLE_NAME = { #else .sbgemm_unroll_mn = MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N), #endif - .sbgemm_align_k = SBGEMM_ALIGN_K, .need_amxtile_permission = 0, // need_amxtile_permission +#endif +#if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) + .sgemm_p = 0, + .sgemm_q = 0, + .sgemm_r = 0, + .sgemm_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .sgemm_unroll_n = SGEMM_DEFAULT_UNROLL_N, +#ifdef SGEMM_DEFAULT_UNROLL_MN + .sgemm_unroll_mn = SGEMM_DEFAULT_UNROLL_MN, +#else + .sgemm_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#ifdef HAVE_EXCLUSIVE_CACHE + .exclusive_cache = 1, +#else + .exclusive_cache = 0, +#endif +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + .dgemm_p = 0, + .dgemm_q = 0, + .dgemm_r = 0, + .dgemm_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .dgemm_unroll_n = DGEMM_DEFAULT_UNROLL_N, +#ifdef DGEMM_DEFAULT_UNROLL_MN + .dgemm_unroll_mn = DGEMM_DEFAULT_UNROLL_MN, +#else + .dgemm_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#ifdef EXPRECISION + .qgemm_p = 0, + .qgemm_q = 0, + .qgemm_r = 0, + .qgemm_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .qgemm_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .qgemm_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), +#endif +#if (BUILD_COMPLEX) + .cgemm_p = 0, + .cgemm_q = 0, + .cgemm_r = 0, + .cgemm_unroll_m = CGEMM_DEFAULT_UNROLL_M, + .cgemm_unroll_n = CGEMM_DEFAULT_UNROLL_N, +#ifdef CGEMM_DEFAULT_UNROLL_MN + .cgemm_unroll_mn = CGEMM_DEFAULT_UNROLL_MN, +#else + .cgemm_unroll_mn = MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), +#endif +#endif +#if (BUILD_COMPLEX) + .cgemm3m_p = 0, + .cgemm3m_q = 0, + .cgemm3m_r = 0, +#if (USE_GEMM3M) +#ifdef CGEMM3M_DEFAULT_UNROLL_M + .cgemm3m_unroll_m = CGEMM3M_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = CGEMM3M_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), +#else + .cgemm3m_unroll_m = SGEMM_DEFAULT_UNROLL_M, + .cgemm3m_unroll_n = SGEMM_DEFAULT_UNROLL_N, + .cgemm3m_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +#endif +#else + .cgemm3m_unroll_m = 0, + .cgemm3m_unroll_n = 0, + .cgemm3m_unroll_mn = 0, +#endif +#endif +#if BUILD_COMPLEX16 == 1 + .zgemm_p = 0, + .zgemm_q = 0, + .zgemm_r = 0, + .zgemm_unroll_m = ZGEMM_DEFAULT_UNROLL_M, + .zgemm_unroll_n = ZGEMM_DEFAULT_UNROLL_N, +#ifdef ZGEMM_DEFAULT_UNROLL_MN + .zgemm_unroll_mn = ZGEMM_DEFAULT_UNROLL_MN, +#else + .zgemm_unroll_mn = MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), +#endif + .zgemm3m_p = 0, + .zgemm3m_q = 0, + .zgemm3m_r = 0, +#if (USE_GEMM3M) +#ifdef ZGEMM3M_DEFAULT_UNROLL_M + .zgemm3m_unroll_m = ZGEMM3M_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = ZGEMM3M_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), +#else + .zgemm3m_unroll_m = DGEMM_DEFAULT_UNROLL_M, + .zgemm3m_unroll_n = DGEMM_DEFAULT_UNROLL_N, + .zgemm3m_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +#endif +#else + .zgemm3m_unroll_m = 0, + .zgemm3m_unroll_n = 0, + .zgemm3m_unroll_mn = 0, +#endif +#endif +#ifdef EXPRECISION + .xgemm_p = 0, + .xgemm_q = 0, + .xgemm_r = 0, + .xgemm_unroll_m = XGEMM_DEFAULT_UNROLL_M, + .xgemm_unroll_n = XGEMM_DEFAULT_UNROLL_N, + .xgemm_unroll_mn = MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), + .xgemm3m_p = 0, + .xgemm3m_q = 0, + .xgemm3m_r = 0, +#if (USE_GEMM3M) + .xgemm3m_unroll_m = QGEMM_DEFAULT_UNROLL_M, + .xgemm3m_unroll_n = QGEMM_DEFAULT_UNROLL_N, + .xgemm3m_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), +#else + .xgemm3m_unroll_m = 0, + .xgemm3m_unroll_n = 0, + .xgemm3m_unroll_mn = 0, +#endif +#endif + .init = init_parameter, + .snum_opt = SNUMOPT, + .dnum_opt = DNUMOPT, + .qnum_opt = QNUMOPT, +}; +#if BUILD_HFLOAT16 == 1 +const openblas_shgemm_dispatch_t openblas_shgemm_dispatchTS = { + .shgemm_kernel = shgemm_kernelTS, + .shgemm_beta = shgemm_betaTS, +#if SHGEMM_DEFAULT_UNROLL_M != SHGEMM_DEFAULT_UNROLL_N + .shgemm_incopy = shgemm_incopyTS, + .shgemm_itcopy = shgemm_itcopyTS, +#else + .shgemm_incopy = shgemm_oncopyTS, + .shgemm_itcopy = shgemm_otcopyTS, +#endif + .shgemm_oncopy = shgemm_oncopyTS, + .shgemm_otcopy = shgemm_otcopyTS, +}; +#endif + +#if BUILD_HFLOAT16 == 1 +const openblas_shgemv_dispatch_t openblas_shgemv_dispatchTS = { + .shgemv_n = shgemv_nTS, + .shgemv_t = shgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbstobf16_dispatch_t openblas_sbstobf16_dispatchTS = { .sbstobf16_k = sbstobf16_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbdtobf16_dispatch_t openblas_sbdtobf16_dispatchTS = { .sbdtobf16_k = sbdtobf16_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbf16tos_dispatch_t openblas_sbf16tos_dispatchTS = { .sbf16tos_k = sbf16tos_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_dbf16tod_dispatch_t openblas_dbf16tod_dispatchTS = { .dbf16tod_k = dbf16tod_kTS, +}; +#endif +#if BUILD_BFLOAT16 == 1 +const openblas_sbamax_dispatch_t openblas_sbamax_dispatchTS = { .sbamax_k = samax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbamin_dispatch_t openblas_sbamin_dispatchTS = { .sbamin_k = samin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbmax_dispatch_t openblas_sbmax_dispatchTS = { .sbmax_k = smax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbmin_dispatch_t openblas_sbmin_dispatchTS = { .sbmin_k = smin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbamax_dispatch_t openblas_isbamax_dispatchTS = { .isbamax_k = isamax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbamin_dispatch_t openblas_isbamin_dispatchTS = { .isbamin_k = isamin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbmax_dispatch_t openblas_isbmax_dispatchTS = { .isbmax_k = ismax_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_isbmin_dispatch_t openblas_isbmin_dispatchTS = { .isbmin_k = ismin_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbnrm2_dispatch_t openblas_sbnrm2_dispatchTS = { .sbnrm2_k = snrm2_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbasum_dispatch_t openblas_sbasum_dispatchTS = { .sbasum_k = sasum_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsum_dispatch_t openblas_sbsum_dispatchTS = { .sbsum_k = ssum_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbcopy_dispatch_t openblas_sbcopy_dispatchTS = { .sbcopy_k = scopy_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbdot_dispatch_t openblas_sbdot_dispatchTS = { .sbdot_k = sbdot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_dsbdot_dispatch_t openblas_dsbdot_dispatchTS = { .dsbdot_k = dsdot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbrot_dispatch_t openblas_sbrot_dispatchTS = { .sbrot_k = srot_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbrotm_dispatch_t openblas_sbrotm_dispatchTS = { .sbrotm_k = srotm_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_bscal_dispatch_t openblas_bscal_dispatchTS = { .bscal_k = bscal_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbaxpy_dispatch_t openblas_sbaxpy_dispatchTS = { .sbaxpy_k = saxpy_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbscal_dispatch_t openblas_sbscal_dispatchTS = { .sbscal_k = sscal_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbswap_dispatch_t openblas_sbswap_dispatchTS = { .sbswap_k = sswap_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_bgemv_dispatch_t openblas_bgemv_dispatchTS = { .bgemv_n = bgemv_nTS, .bgemv_t = bgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbgemv_dispatch_t openblas_sbgemv_dispatchTS = { .sbgemv_n = sbgemv_nTS, .sbgemv_t = sbgemv_tTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbger_dispatch_t openblas_sbger_dispatchTS = { .sbger_k = sger_kTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsymv_dispatch_t openblas_sbsymv_dispatchTS = { .sbsymv_L = ssymv_LTS, .sbsymv_U = ssymv_UTS, +}; +#endif +#if BUILD_BFLOAT16 == 1 +const openblas_bgemm_dispatch_t openblas_bgemm_dispatchTS = { .bgemm_kernel = bgemm_kernelTS, .bgemm_beta = bgemm_betaTS, #if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N @@ -164,7 +429,11 @@ gotoblas_t TABLE_NAME = { #endif .bgemm_oncopy = bgemm_oncopyTS, .bgemm_otcopy = bgemm_otcopyTS, +}; +#endif +#if BUILD_BFLOAT16 == 1 +const openblas_sbgemm_dispatch_t openblas_sbgemm_dispatchTS = { .sbgemm_kernel = sbgemm_kernelTS, .sbgemm_beta = sbgemm_betaTS, #if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N @@ -176,7 +445,22 @@ gotoblas_t TABLE_NAME = { #endif .sbgemm_oncopy = sbgemm_oncopyTS, .sbgemm_otcopy = sbgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .sbgemm_small_matrix_permit = sbgemm_small_matrix_permitTS, + .sbgemm_small_kernel_nn = sbgemm_small_kernel_nnTS, + .sbgemm_small_kernel_nt = sbgemm_small_kernel_ntTS, + .sbgemm_small_kernel_tn = sbgemm_small_kernel_tnTS, + .sbgemm_small_kernel_tt = sbgemm_small_kernel_ttTS, + .sbgemm_small_kernel_b0_nn = sbgemm_small_kernel_b0_nnTS, + .sbgemm_small_kernel_b0_nt = sbgemm_small_kernel_b0_ntTS, + .sbgemm_small_kernel_b0_tn = sbgemm_small_kernel_b0_tnTS, + .sbgemm_small_kernel_b0_tt = sbgemm_small_kernel_b0_ttTS, +#endif +}; +#endif +#if BUILD_BFLOAT16 == 1 +const openblas_sbtrsm_dispatch_t openblas_sbtrsm_dispatchTS = { .sbtrsm_kernel_LN = strsm_kernel_LNTS, .sbtrsm_kernel_LT = strsm_kernel_LTTS, .sbtrsm_kernel_RN = strsm_kernel_RNTS, @@ -208,6 +492,11 @@ gotoblas_t TABLE_NAME = { .sbtrsm_olnncopy = strsm_olnncopyTS, .sbtrsm_oltucopy = strsm_oltucopyTS, .sbtrsm_oltncopy = strsm_oltncopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbtrmm_dispatch_t openblas_sbtrmm_dispatchTS = { .sbtrmm_kernel_RN = strmm_kernel_RNTS, .sbtrmm_kernel_RT = strmm_kernel_RTTS, .sbtrmm_kernel_LN = strmm_kernel_LNTS, @@ -239,6 +528,11 @@ gotoblas_t TABLE_NAME = { .sbtrmm_olnncopy = strmm_olnncopyTS, .sbtrmm_oltucopy = strmm_oltucopyTS, .sbtrmm_oltncopy = strmm_oltncopyTS, +}; +#endif + +#if BUILD_BFLOAT16 == 1 +const openblas_sbsymm_dispatch_t openblas_sbsymm_dispatchTS = { #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N .sbsymm_iutcopy = ssymm_iutcopyTS, .sbsymm_iltcopy = ssymm_iltcopyTS, @@ -248,91 +542,159 @@ gotoblas_t TABLE_NAME = { #endif .sbsymm_outcopy = ssymm_outcopyTS, .sbsymm_oltcopy = ssymm_oltcopyTS, +}; +#endif +#if BUILD_BFLOAT16 == 1 +const openblas_sbneg_dispatch_t openblas_sbneg_dispatchTS = { #ifndef NO_LAPACK .sbneg_tcopy = sneg_tcopyTS, - .sblaswp_ncopy = slaswp_ncopyTS, #else .sbneg_tcopy = NULL, - .sblaswp_ncopy = NULL, -#endif -#ifdef SMALL_MATRIX_OPT - .sbgemm_small_matrix_permit = sbgemm_small_matrix_permitTS, - .sbgemm_small_kernel_nn = sbgemm_small_kernel_nnTS, - .sbgemm_small_kernel_nt = sbgemm_small_kernel_ntTS, - .sbgemm_small_kernel_tn = sbgemm_small_kernel_tnTS, - .sbgemm_small_kernel_tt = sbgemm_small_kernel_ttTS, - .sbgemm_small_kernel_b0_nn = sbgemm_small_kernel_b0_nnTS, - .sbgemm_small_kernel_b0_nt = sbgemm_small_kernel_b0_ntTS, - .sbgemm_small_kernel_b0_tn = sbgemm_small_kernel_b0_tnTS, - .sbgemm_small_kernel_b0_tt = sbgemm_small_kernel_b0_ttTS, #endif +}; #endif -#if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) - .sgemm_p = 0, - .sgemm_q = 0, - .sgemm_r = 0, - .sgemm_unroll_m = SGEMM_DEFAULT_UNROLL_M, - .sgemm_unroll_n = SGEMM_DEFAULT_UNROLL_N, -#ifdef SGEMM_DEFAULT_UNROLL_MN - .sgemm_unroll_mn = SGEMM_DEFAULT_UNROLL_MN, +#if BUILD_BFLOAT16 == 1 +const openblas_sblaswp_dispatch_t openblas_sblaswp_dispatchTS = { +#ifndef NO_LAPACK + .sblaswp_ncopy = slaswp_ncopyTS, #else - .sgemm_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), + .sblaswp_ncopy = NULL, #endif +}; #endif -#ifdef HAVE_EXCLUSIVE_CACHE - .exclusive_cache = 1, -#else - .exclusive_cache = 0, +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_samax_dispatch_t openblas_samax_dispatchTS = { + .samax_k = samax_kTS, +}; #endif -#if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) - .samax_k = samax_kTS, +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_samin_dispatch_t openblas_samin_dispatchTS = { .samin_k = samin_kTS, +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_smax_dispatch_t openblas_smax_dispatchTS = { .smax_k = smax_kTS, +}; +#endif + +#if (BUILD_SINGLE == 1) || (BUILD_COMPLEX == 1) +const openblas_smin_dispatch_t openblas_smin_dispatchTS = { .smin_k = smin_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE ==1) || (BUILD_COMPLEX==1) +const openblas_isamax_dispatch_t openblas_isamax_dispatchTS = { .isamax_k = isamax_kTS, +}; #endif -#if (BUILD_SINGLE==1 ) || (BUILD_COMPLEX==1) + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_isamin_dispatch_t openblas_isamin_dispatchTS = { .isamin_k = isamin_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_ismax_dispatch_t openblas_ismax_dispatchTS = { .ismax_k = ismax_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_ismin_dispatch_t openblas_ismin_dispatchTS = { .ismin_k = ismin_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_snrm2_dispatch_t openblas_snrm2_dispatchTS = { .snrm2_k = snrm2_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) +const openblas_sasum_dispatch_t openblas_sasum_dispatchTS = { .sasum_k = sasum_kTS, +}; #endif -#if BUILD_SINGLE == 1 + +#if (BUILD_SINGLE==1) +const openblas_ssum_dispatch_t openblas_ssum_dispatchTS = { .ssum_k = ssum_kTS, +}; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_scopy_dispatch_t openblas_scopy_dispatchTS = { .scopy_k = scopy_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sdot_dispatch_t openblas_sdot_dispatchTS = { .sdot_k = sdot_kTS, -// dsdot_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_srot_dispatch_t openblas_srot_dispatchTS = { .srot_k = srot_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_srotm_dispatch_t openblas_srotm_dispatchTS = { .srotm_k = srotm_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_saxpy_dispatch_t openblas_saxpy_dispatchTS = { .saxpy_k = saxpy_kTS, +}; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1) +const openblas_sscal_dispatch_t openblas_sscal_dispatchTS = { .sscal_k = sscal_kTS, +}; #endif + #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sswap_dispatch_t openblas_sswap_dispatchTS = { .sswap_k = sswap_kTS, +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sgemv_dispatch_t openblas_sgemv_dispatchTS = { .sgemv_n = sgemv_nTS, .sgemv_t = sgemv_tTS, +}; #endif -#if BUILD_SINGLE == 1 + +#if (BUILD_SINGLE==1) +const openblas_sger_dispatch_t openblas_sger_dispatchTS = { .sger_k = sger_kTS, +}; #endif -#if BUILD_SINGLE == 1 + +#if (BUILD_SINGLE==1) +const openblas_ssymv_dispatch_t openblas_ssymv_dispatchTS = { .ssymv_L = ssymv_LTS, .ssymv_U = ssymv_UTS, +}; #endif #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_sgemm_dispatch_t openblas_sgemm_dispatchTS = { #ifdef ARCH_X86_64 .sgemm_direct = sgemm_directTS, .sgemm_direct_performant = sgemm_direct_performantTS, @@ -341,57 +703,131 @@ gotoblas_t TABLE_NAME = { .sgemm_direct = sgemm_directTS, .sgemm_direct_performant = sgemm_direct_performantTS, .sgemm_direct_alpha_beta = sgemm_direct_alpha_betaTS, +#ifdef HAVE_SME + .sme_sgemm_kernel = sme_sgemm_kernelTS, +#else + .sme_sgemm_kernel = NULL, +#endif +#endif + .sgemm_kernel = sgemm_kernelTS, + .sgemm_beta = sgemm_betaTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .sgemm_incopy = sgemm_incopyTS, + .sgemm_itcopy = sgemm_itcopyTS, +#else + .sgemm_incopy = sgemm_oncopyTS, + .sgemm_itcopy = sgemm_otcopyTS, +#endif + .sgemm_oncopy = sgemm_oncopyTS, + .sgemm_otcopy = sgemm_otcopyTS, +#ifdef SMALL_MATRIX_OPT + .sgemm_small_matrix_permit = sgemm_small_matrix_permitTS, + .sgemm_small_kernel_nn = sgemm_small_kernel_nnTS, + .sgemm_small_kernel_nt = sgemm_small_kernel_ntTS, + .sgemm_small_kernel_tn = sgemm_small_kernel_tnTS, + .sgemm_small_kernel_tt = sgemm_small_kernel_ttTS, + .sgemm_small_kernel_b0_nn = sgemm_small_kernel_b0_nnTS, + .sgemm_small_kernel_b0_nt = sgemm_small_kernel_b0_ntTS, + .sgemm_small_kernel_b0_tn = sgemm_small_kernel_b0_tnTS, + .sgemm_small_kernel_b0_tt = sgemm_small_kernel_b0_ttTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_ssymm_dispatch_t openblas_ssymm_dispatchTS = { +#ifdef ARCH_ARM64 .ssymm_direct_alpha_betaLU = ssymm_direct_alpha_betaLUTS, .ssymm_direct_alpha_betaLL = ssymm_direct_alpha_betaLLTS, +#endif + .ssymm_kernel = ssymm_kernelTS, +#if (BUILD_SINGLE==1) + .ssymm_incopy = ssymm_incopyTS, + .ssymm_itcopy = ssymm_itcopyTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .ssymm_iutcopy = ssymm_iutcopyTS, + .ssymm_iltcopy = ssymm_iltcopyTS, +#else + .ssymm_iutcopy = ssymm_outcopyTS, + .ssymm_iltcopy = ssymm_oltcopyTS, +#endif + .ssymm_outcopy = ssymm_outcopyTS, + .ssymm_oltcopy = ssymm_oltcopyTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_strmm_dispatch_t openblas_strmm_dispatchTS = { +#ifdef ARCH_ARM64 .strmm_direct_LNUN = strmm_direct_LNUNTS, .strmm_direct_LNLN = strmm_direct_LNLNTS, .strmm_direct_LTUN = strmm_direct_LTUNTS, .strmm_direct_LTLN = strmm_direct_LTLNTS, +#endif + .strmm_gemm_kernel = strmm_gemm_kernelTS, +#if (BUILD_SINGLE==1) + .strmm_kernel_RN = strmm_kernel_RNTS, + .strmm_kernel_RT = strmm_kernel_RTTS, + .strmm_kernel_LN = strmm_kernel_LNTS, + .strmm_kernel_LT = strmm_kernel_LTTS, +#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N + .strmm_iunucopy = strmm_iunucopyTS, + .strmm_iunncopy = strmm_iunncopyTS, + .strmm_iutucopy = strmm_iutucopyTS, + .strmm_iutncopy = strmm_iutncopyTS, + .strmm_ilnucopy = strmm_ilnucopyTS, + .strmm_ilnncopy = strmm_ilnncopyTS, + .strmm_iltucopy = strmm_iltucopyTS, + .strmm_iltncopy = strmm_iltncopyTS, +#else + .strmm_iunucopy = strmm_ounucopyTS, + .strmm_iunncopy = strmm_ounncopyTS, + .strmm_iutucopy = strmm_outucopyTS, + .strmm_iutncopy = strmm_outncopyTS, + .strmm_ilnucopy = strmm_olnucopyTS, + .strmm_ilnncopy = strmm_olnncopyTS, + .strmm_iltucopy = strmm_oltucopyTS, + .strmm_iltncopy = strmm_oltncopyTS, +#endif + .strmm_incopy = strmm_incopyTS, + .strmm_itcopy = strmm_itcopyTS, + .strmm_ounucopy = strmm_ounucopyTS, + .strmm_ounncopy = strmm_ounncopyTS, + .strmm_outucopy = strmm_outucopyTS, + .strmm_outncopy = strmm_outncopyTS, + .strmm_olnucopy = strmm_olnucopyTS, + .strmm_olnncopy = strmm_olnncopyTS, + .strmm_oltucopy = strmm_oltucopyTS, + .strmm_oltncopy = strmm_oltncopyTS, +#endif +}; +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +const openblas_ssyrk_dispatch_t openblas_ssyrk_dispatchTS = { .ssyrk_direct_alpha_betaUN = ssyrk_direct_alpha_betaUNTS, .ssyrk_direct_alpha_betaUT = ssyrk_direct_alpha_betaUTTS, .ssyrk_direct_alpha_betaLN = ssyrk_direct_alpha_betaLNTS, .ssyrk_direct_alpha_betaLT = ssyrk_direct_alpha_betaLTTS, +}; +#endif +#endif + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +#ifdef ARCH_ARM64 +const openblas_ssyr2k_dispatch_t openblas_ssyr2k_dispatchTS = { .ssyr2k_direct_alpha_betaUN = ssyr2k_direct_alpha_betaUNTS, .ssyr2k_direct_alpha_betaUT = ssyr2k_direct_alpha_betaUTTS, .ssyr2k_direct_alpha_betaLN = ssyr2k_direct_alpha_betaLNTS, .ssyr2k_direct_alpha_betaLT = ssyr2k_direct_alpha_betaLTTS, -#ifdef HAVE_SME - .sme_sgemm_kernel = sme_sgemm_kernelTS, -#else - .sme_sgemm_kernel = NULL, -#endif -#endif - - .sgemm_kernel = sgemm_kernelTS, - .sgemm_beta = sgemm_betaTS, - .ssymm_kernel = ssymm_kernelTS, - .strmm_gemm_kernel = strmm_gemm_kernelTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - .sgemm_incopy = sgemm_incopyTS, - .sgemm_itcopy = sgemm_itcopyTS, -#else - .sgemm_incopy = sgemm_oncopyTS, - .sgemm_itcopy = sgemm_otcopyTS, -#endif - .sgemm_oncopy = sgemm_oncopyTS, - .sgemm_otcopy = sgemm_otcopyTS, -#endif - -#if BUILD_SINGLE == 1 || BUILD_DOUBLE == 1 || BUILD_COMPLEX == 1 -#ifdef SMALL_MATRIX_OPT - .sgemm_small_matrix_permit = sgemm_small_matrix_permitTS, - .sgemm_small_kernel_nn = sgemm_small_kernel_nnTS, - .sgemm_small_kernel_nt = sgemm_small_kernel_ntTS, - .sgemm_small_kernel_tn = sgemm_small_kernel_tnTS, - .sgemm_small_kernel_tt = sgemm_small_kernel_ttTS, - .sgemm_small_kernel_b0_nn = sgemm_small_kernel_b0_nnTS, - .sgemm_small_kernel_b0_nt = sgemm_small_kernel_b0_ntTS, - .sgemm_small_kernel_b0_tn = sgemm_small_kernel_b0_tnTS, - .sgemm_small_kernel_b0_tt = sgemm_small_kernel_b0_ttTS, +}; #endif #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX == 1) +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) +const openblas_strsm_dispatch_t openblas_strsm_dispatchTS = { .strsm_kernel_LN = strsm_kernel_LNTS, .strsm_kernel_LT = strsm_kernel_LTTS, .strsm_kernel_RN = strsm_kernel_RNTS, @@ -423,113 +859,165 @@ gotoblas_t TABLE_NAME = { .strsm_olnncopy = strsm_olnncopyTS, .strsm_oltucopy = strsm_oltucopyTS, .strsm_oltncopy = strsm_oltncopyTS, +}; #endif + #if (BUILD_SINGLE==1) - .strmm_kernel_RN = strmm_kernel_RNTS, - .strmm_kernel_RT = strmm_kernel_RTTS, - .strmm_kernel_LN = strmm_kernel_LNTS, - .strmm_kernel_LT = strmm_kernel_LTTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - .strmm_iunucopy = strmm_iunucopyTS, - .strmm_iunncopy = strmm_iunncopyTS, - .strmm_iutucopy = strmm_iutucopyTS, - .strmm_iutncopy = strmm_iutncopyTS, - .strmm_ilnucopy = strmm_ilnucopyTS, - .strmm_ilnncopy = strmm_ilnncopyTS, - .strmm_iltucopy = strmm_iltucopyTS, - .strmm_iltncopy = strmm_iltncopyTS, -#else - .strmm_iunucopy = strmm_ounucopyTS, - .strmm_iunncopy = strmm_ounncopyTS, - .strmm_iutucopy = strmm_outucopyTS, - .strmm_iutncopy = strmm_outncopyTS, - .strmm_ilnucopy = strmm_olnucopyTS, - .strmm_ilnncopy = strmm_olnncopyTS, - .strmm_iltucopy = strmm_oltucopyTS, - .strmm_iltncopy = strmm_oltncopyTS, -#endif - .ssymm_incopy = ssymm_incopyTS, - .ssymm_itcopy = ssymm_itcopyTS, - .strmm_incopy = strmm_incopyTS, - .strmm_itcopy = strmm_itcopyTS, - .strmm_ounucopy = strmm_ounucopyTS, - .strmm_ounncopy = strmm_ounncopyTS, - .strmm_outucopy = strmm_outucopyTS, - .strmm_outncopy = strmm_outncopyTS, - .strmm_olnucopy = strmm_olnucopyTS, - .strmm_olnncopy = strmm_olnncopyTS, - .strmm_oltucopy = strmm_oltucopyTS, - .strmm_oltncopy = strmm_oltncopyTS, -#if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N - .ssymm_iutcopy = ssymm_iutcopyTS, - .ssymm_iltcopy = ssymm_iltcopyTS, -#else - .ssymm_iutcopy = ssymm_outcopyTS, - .ssymm_iltcopy = ssymm_oltcopyTS, -#endif - .ssymm_outcopy = ssymm_outcopyTS, - .ssymm_oltcopy = ssymm_oltcopyTS, +const openblas_sneg_dispatch_t openblas_sneg_dispatchTS = { #ifndef NO_LAPACK .sneg_tcopy = sneg_tcopyTS, - .slaswp_ncopy = slaswp_ncopyTS, #else .sneg_tcopy = NULL, - .slaswp_ncopy = NULL, #endif +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) - .dgemm_p = 0, - .dgemm_q = 0, - .dgemm_r = 0, - .dgemm_unroll_m = DGEMM_DEFAULT_UNROLL_M, - .dgemm_unroll_n = DGEMM_DEFAULT_UNROLL_N, -#ifdef DGEMM_DEFAULT_UNROLL_MN - .dgemm_unroll_mn = DGEMM_DEFAULT_UNROLL_MN, +#if (BUILD_SINGLE==1) +const openblas_slaswp_dispatch_t openblas_slaswp_dispatchTS = { +#ifndef NO_LAPACK + .slaswp_ncopy = slaswp_ncopyTS, #else - .dgemm_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), + .slaswp_ncopy = NULL, #endif +}; #endif - -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_damax_dispatch_t openblas_damax_dispatchTS = { .damax_k = damax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_damin_dispatch_t openblas_damin_dispatchTS = { .damin_k = damin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dmax_dispatch_t openblas_dmax_dispatchTS = { .dmax_k = dmax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dmin_dispatch_t openblas_dmin_dispatchTS = { .dmin_k = dmin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idamax_dispatch_t openblas_idamax_dispatchTS = { .idamax_k = idamax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idamin_dispatch_t openblas_idamin_dispatchTS = { .idamin_k = idamin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idmax_dispatch_t openblas_idmax_dispatchTS = { .idmax_k = idmax_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_idmin_dispatch_t openblas_idmin_dispatchTS = { .idmin_k = idmin_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dnrm2_dispatch_t openblas_dnrm2_dispatchTS = { .dnrm2_k = dnrm2_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dasum_dispatch_t openblas_dasum_dispatchTS = { .dasum_k = dasum_kTS, +}; #endif -#if (BUILD_DOUBLE==1) + +#if (BUILD_DOUBLE==1) +const openblas_dsum_dispatch_t openblas_dsum_dispatchTS = { .dsum_k = dsum_kTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dcopy_dispatch_t openblas_dcopy_dispatchTS = { .dcopy_k = dcopy_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_ddot_dispatch_t openblas_ddot_dispatchTS = { .ddot_k = ddot_kTS, +}; #endif -#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) + +#if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) +const openblas_dsdot_dispatch_t openblas_dsdot_dispatchTS = { .dsdot_k = dsdot_kTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_drot_dispatch_t openblas_drot_dispatchTS = { .drot_k = drot_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_drotm_dispatch_t openblas_drotm_dispatchTS = { .drotm_k = drotm_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_daxpy_dispatch_t openblas_daxpy_dispatchTS = { .daxpy_k = daxpy_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dscal_dispatch_t openblas_dscal_dispatchTS = { .dscal_k = dscal_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dswap_dispatch_t openblas_dswap_dispatchTS = { .dswap_k = dswap_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dgemv_dispatch_t openblas_dgemv_dispatchTS = { .dgemv_n = dgemv_nTS, .dgemv_t = dgemv_tTS, +}; #endif -#if (BUILD_DOUBLE==1) + +#if (BUILD_DOUBLE==1) +const openblas_dger_dispatch_t openblas_dger_dispatchTS = { .dger_k = dger_kTS, +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dsymv_dispatch_t openblas_dsymv_dispatchTS = { .dsymv_L = dsymv_LTS, .dsymv_U = dsymv_UTS, +}; #endif -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dgemm_dispatch_t openblas_dgemm_dispatchTS = { #ifdef ARCH_ARM64 #ifdef HAVE_SME .sme_dgemm_kernel = sme_dgemm_kernelTS, @@ -539,8 +1027,6 @@ gotoblas_t TABLE_NAME = { #endif .dgemm_kernel = dgemm_kernelTS, .dgemm_beta = dgemm_betaTS, - .dsymm_kernel = dsymm_kernelTS, - .dtrmm_gemm_kernel = dtrmm_gemm_kernelTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N .dgemm_incopy = dgemm_incopyTS, .dgemm_itcopy = dgemm_itcopyTS, @@ -550,9 +1036,6 @@ gotoblas_t TABLE_NAME = { #endif .dgemm_oncopy = dgemm_oncopyTS, .dgemm_otcopy = dgemm_otcopyTS, -#endif - -#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) #ifdef SMALL_MATRIX_OPT .dgemm_small_matrix_permit = dgemm_small_matrix_permitTS, .dgemm_small_kernel_nn = dgemm_small_kernel_nnTS, @@ -564,39 +1047,32 @@ gotoblas_t TABLE_NAME = { .dgemm_small_kernel_b0_tn = dgemm_small_kernel_b0_tnTS, .dgemm_small_kernel_b0_tt = dgemm_small_kernel_b0_ttTS, #endif +}; #endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dsymm_dispatch_t openblas_dsymm_dispatchTS = { + .dsymm_kernel = dsymm_kernelTS, #if (BUILD_DOUBLE==1) - .dtrsm_kernel_LN = dtrsm_kernel_LNTS, - .dtrsm_kernel_LT = dtrsm_kernel_LTTS, - .dtrsm_kernel_RN = dtrsm_kernel_RNTS, - .dtrsm_kernel_RT = dtrsm_kernel_RTTS, + .dsymm_incopy = dsymm_incopyTS, + .dsymm_itcopy = dsymm_itcopyTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - .dtrsm_iunucopy = dtrsm_iunucopyTS, - .dtrsm_iunncopy = dtrsm_iunncopyTS, - .dtrsm_iutucopy = dtrsm_iutucopyTS, - .dtrsm_iutncopy = dtrsm_iutncopyTS, - .dtrsm_ilnucopy = dtrsm_ilnucopyTS, - .dtrsm_ilnncopy = dtrsm_ilnncopyTS, - .dtrsm_iltucopy = dtrsm_iltucopyTS, - .dtrsm_iltncopy = dtrsm_iltncopyTS, + .dsymm_iutcopy = dsymm_iutcopyTS, + .dsymm_iltcopy = dsymm_iltcopyTS, #else - .dtrsm_iunucopy = dtrsm_ounucopyTS, - .dtrsm_iunncopy = dtrsm_ounncopyTS, - .dtrsm_iutucopy = dtrsm_outucopyTS, - .dtrsm_iutncopy = dtrsm_outncopyTS, - .dtrsm_ilnucopy = dtrsm_olnucopyTS, - .dtrsm_ilnncopy = dtrsm_olnncopyTS, - .dtrsm_iltucopy = dtrsm_oltucopyTS, - .dtrsm_iltncopy = dtrsm_oltncopyTS, + .dsymm_iutcopy = dsymm_outcopyTS, + .dsymm_iltcopy = dsymm_oltcopyTS, #endif - .dtrsm_ounucopy = dtrsm_ounucopyTS, - .dtrsm_ounncopy = dtrsm_ounncopyTS, - .dtrsm_outucopy = dtrsm_outucopyTS, - .dtrsm_outncopy = dtrsm_outncopyTS, - .dtrsm_olnucopy = dtrsm_olnucopyTS, - .dtrsm_olnncopy = dtrsm_olnncopyTS, - .dtrsm_oltucopy = dtrsm_oltucopyTS, - .dtrsm_oltncopy = dtrsm_oltncopyTS, + .dsymm_outcopy = dsymm_outcopyTS, + .dsymm_oltcopy = dsymm_oltcopyTS, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) || (BUILD_COMPLEX16==1) +const openblas_dtrmm_dispatch_t openblas_dtrmm_dispatchTS = { + .dtrmm_gemm_kernel = dtrmm_gemm_kernelTS, +#if (BUILD_DOUBLE==1) .dtrmm_kernel_RN = dtrmm_kernel_RNTS, .dtrmm_kernel_RT = dtrmm_kernel_RTTS, .dtrmm_kernel_LN = dtrmm_kernel_LNTS, @@ -620,8 +1096,6 @@ gotoblas_t TABLE_NAME = { .dtrmm_iltucopy = dtrmm_oltucopyTS, .dtrmm_iltncopy = dtrmm_oltncopyTS, #endif - .dsymm_incopy = dsymm_incopyTS, - .dsymm_itcopy = dsymm_itcopyTS, .dtrmm_incopy = dtrmm_incopyTS, .dtrmm_itcopy = dtrmm_itcopyTS, .dtrmm_ounucopy = dtrmm_ounucopyTS, @@ -632,58 +1106,196 @@ gotoblas_t TABLE_NAME = { .dtrmm_olnncopy = dtrmm_olnncopyTS, .dtrmm_oltucopy = dtrmm_oltucopyTS, .dtrmm_oltncopy = dtrmm_oltncopyTS, +#endif +}; +#endif + +#if (BUILD_DOUBLE==1) +const openblas_dtrsm_dispatch_t openblas_dtrsm_dispatchTS = { + .dtrsm_kernel_LN = dtrsm_kernel_LNTS, + .dtrsm_kernel_LT = dtrsm_kernel_LTTS, + .dtrsm_kernel_RN = dtrsm_kernel_RNTS, + .dtrsm_kernel_RT = dtrsm_kernel_RTTS, #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N - .dsymm_iutcopy = dsymm_iutcopyTS, - .dsymm_iltcopy = dsymm_iltcopyTS, + .dtrsm_iunucopy = dtrsm_iunucopyTS, + .dtrsm_iunncopy = dtrsm_iunncopyTS, + .dtrsm_iutucopy = dtrsm_iutucopyTS, + .dtrsm_iutncopy = dtrsm_iutncopyTS, + .dtrsm_ilnucopy = dtrsm_ilnucopyTS, + .dtrsm_ilnncopy = dtrsm_ilnncopyTS, + .dtrsm_iltucopy = dtrsm_iltucopyTS, + .dtrsm_iltncopy = dtrsm_iltncopyTS, #else - .dsymm_iutcopy = dsymm_outcopyTS, - .dsymm_iltcopy = dsymm_oltcopyTS, + .dtrsm_iunucopy = dtrsm_ounucopyTS, + .dtrsm_iunncopy = dtrsm_ounncopyTS, + .dtrsm_iutucopy = dtrsm_outucopyTS, + .dtrsm_iutncopy = dtrsm_outncopyTS, + .dtrsm_ilnucopy = dtrsm_olnucopyTS, + .dtrsm_ilnncopy = dtrsm_olnncopyTS, + .dtrsm_iltucopy = dtrsm_oltucopyTS, + .dtrsm_iltncopy = dtrsm_oltncopyTS, +#endif + .dtrsm_ounucopy = dtrsm_ounucopyTS, + .dtrsm_ounncopy = dtrsm_ounncopyTS, + .dtrsm_outucopy = dtrsm_outucopyTS, + .dtrsm_outncopy = dtrsm_outncopyTS, + .dtrsm_olnucopy = dtrsm_olnucopyTS, + .dtrsm_olnncopy = dtrsm_olnncopyTS, + .dtrsm_oltucopy = dtrsm_oltucopyTS, + .dtrsm_oltncopy = dtrsm_oltncopyTS, +}; #endif - .dsymm_outcopy = dsymm_outcopyTS, - .dsymm_oltcopy = dsymm_oltcopyTS, +#if (BUILD_DOUBLE==1) +const openblas_dneg_dispatch_t openblas_dneg_dispatchTS = { #ifndef NO_LAPACK .dneg_tcopy = dneg_tcopyTS, - .dlaswp_ncopy = dlaswp_ncopyTS, #else .dneg_tcopy = NULL, - .dlaswp_ncopy = NULL, +#endif +}; #endif +#if (BUILD_DOUBLE==1) +const openblas_dlaswp_dispatch_t openblas_dlaswp_dispatchTS = { +#ifndef NO_LAPACK + .dlaswp_ncopy = dlaswp_ncopyTS, +#else + .dlaswp_ncopy = NULL, +#endif +}; #endif #ifdef EXPRECISION - - .qgemm_p = 0, - .qgemm_q = 0, - .qgemm_r = 0, - .qgemm_unroll_m = QGEMM_DEFAULT_UNROLL_M, - .qgemm_unroll_n = QGEMM_DEFAULT_UNROLL_N, - .qgemm_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), - +const openblas_qamax_dispatch_t openblas_qamax_dispatchTS = { .qamax_k = qamax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qamin_dispatch_t openblas_qamin_dispatchTS = { .qamin_k = qamin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qmax_dispatch_t openblas_qmax_dispatchTS = { .qmax_k = qmax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qmin_dispatch_t openblas_qmin_dispatchTS = { .qmin_k = qmin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqamax_dispatch_t openblas_iqamax_dispatchTS = { .iqamax_k = iqamax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqamin_dispatch_t openblas_iqamin_dispatchTS = { .iqamin_k = iqamin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqmax_dispatch_t openblas_iqmax_dispatchTS = { .iqmax_k = iqmax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_iqmin_dispatch_t openblas_iqmin_dispatchTS = { .iqmin_k = iqmin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qnrm2_dispatch_t openblas_qnrm2_dispatchTS = { .qnrm2_k = qnrm2_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qasum_dispatch_t openblas_qasum_dispatchTS = { .qasum_k = qasum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qsum_dispatch_t openblas_qsum_dispatchTS = { .qsum_k = qsum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qcopy_dispatch_t openblas_qcopy_dispatchTS = { .qcopy_k = qcopy_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qdot_dispatch_t openblas_qdot_dispatchTS = { .qdot_k = qdot_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qrot_dispatch_t openblas_qrot_dispatchTS = { .qrot_k = qrot_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qrotm_dispatch_t openblas_qrotm_dispatchTS = { .qrotm_k = qrotm_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qaxpy_dispatch_t openblas_qaxpy_dispatchTS = { .qaxpy_k = qaxpy_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qscal_dispatch_t openblas_qscal_dispatchTS = { .qscal_k = qscal_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qswap_dispatch_t openblas_qswap_dispatchTS = { .qswap_k = qswap_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qgemv_dispatch_t openblas_qgemv_dispatchTS = { .qgemv_n = qgemv_nTS, .qgemv_t = qgemv_tTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qger_dispatch_t openblas_qger_dispatchTS = { .qger_k = qger_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qsymv_dispatch_t openblas_qsymv_dispatchTS = { .qsymv_L = qsymv_LTS, .qsymv_U = qsymv_UTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qgemm_dispatch_t openblas_qgemm_dispatchTS = { .qgemm_kernel = qgemm_kernelTS, .qgemm_beta = qgemm_betaTS, #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N @@ -695,6 +1307,11 @@ gotoblas_t TABLE_NAME = { #endif .qgemm_oncopy = qgemm_oncopyTS, .qgemm_otcopy = qgemm_otcopyTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qtrsm_dispatch_t openblas_qtrsm_dispatchTS = { .qtrsm_kernel_LN = qtrsm_kernel_LNTS, .qtrsm_kernel_LT = qtrsm_kernel_LTTS, .qtrsm_kernel_RN = qtrsm_kernel_RNTS, @@ -726,6 +1343,11 @@ gotoblas_t TABLE_NAME = { .qtrsm_olnncopy = qtrsm_olnncopyTS, .qtrsm_oltucopy = qtrsm_oltucopyTS, .qtrsm_oltncopy = qtrsm_oltncopyTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qtrmm_dispatch_t openblas_qtrmm_dispatchTS = { .qtrmm_kernel_RN = qtrmm_kernel_RNTS, .qtrmm_kernel_RT = qtrmm_kernel_RTTS, .qtrmm_kernel_LN = qtrmm_kernel_LNTS, @@ -757,6 +1379,11 @@ gotoblas_t TABLE_NAME = { .qtrmm_olnncopy = qtrmm_olnncopyTS, .qtrmm_oltucopy = qtrmm_oltucopyTS, .qtrmm_oltncopy = qtrmm_oltncopyTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_qsymm_dispatch_t openblas_qsymm_dispatchTS = { #if QGEMM_DEFAULT_UNROLL_M != QGEMM_DEFAULT_UNROLL_N .qsymm_iutcopy = qsymm_iutcopyTS, .qsymm_iltcopy = qsymm_iltcopyTS, @@ -766,55 +1393,121 @@ gotoblas_t TABLE_NAME = { #endif .qsymm_outcopy = qsymm_outcopyTS, .qsymm_oltcopy = qsymm_oltcopyTS, +}; +#endif +#ifdef EXPRECISION +const openblas_qneg_dispatch_t openblas_qneg_dispatchTS = { #ifndef NO_LAPACK .qneg_tcopy = qneg_tcopyTS, - .qlaswp_ncopy = qlaswp_ncopyTS, #else .qneg_tcopy = NULL, - .qlaswp_ncopy = NULL, #endif - +}; #endif -#if (BUILD_COMPLEX) - .cgemm_p = 0, - .cgemm_q = 0, - .cgemm_r = 0, - .cgemm_unroll_m = CGEMM_DEFAULT_UNROLL_M, - .cgemm_unroll_n = CGEMM_DEFAULT_UNROLL_N, -#ifdef CGEMM_DEFAULT_UNROLL_MN - .cgemm_unroll_mn = CGEMM_DEFAULT_UNROLL_MN, +#ifdef EXPRECISION +const openblas_qlaswp_dispatch_t openblas_qlaswp_dispatchTS = { +#ifndef NO_LAPACK + .qlaswp_ncopy = qlaswp_ncopyTS, #else - .cgemm_unroll_mn = MAX(CGEMM_DEFAULT_UNROLL_M, CGEMM_DEFAULT_UNROLL_N), + .qlaswp_ncopy = NULL, #endif -#if (BUILD_COMPLEX) +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_camax_dispatch_t openblas_camax_dispatchTS = { .camax_k = camax_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_camin_dispatch_t openblas_camin_dispatchTS = { .camin_k = camin_kTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_icamax_dispatch_t openblas_icamax_dispatchTS = { .icamax_k = icamax_kTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_icamin_dispatch_t openblas_icamin_dispatchTS = { .icamin_k = icamin_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cnrm2_dispatch_t openblas_cnrm2_dispatchTS = { .cnrm2_k = cnrm2_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_casum_dispatch_t openblas_casum_dispatchTS = { .casum_k = casum_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_csum_dispatch_t openblas_csum_dispatchTS = { .csum_k = csum_kTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_ccopy_dispatch_t openblas_ccopy_dispatchTS = { .ccopy_k = ccopy_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cdotu_dispatch_t openblas_cdotu_dispatchTS = { .cdotu_k = cdotu_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cdotc_dispatch_t openblas_cdotc_dispatchTS = { .cdotc_k = cdotc_kTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_csrot_dispatch_t openblas_csrot_dispatchTS = { .csrot_k = csrot_kTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_caxpy_dispatch_t openblas_caxpy_dispatchTS = { .caxpy_k = caxpy_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_caxpyc_dispatch_t openblas_caxpyc_dispatchTS = { .caxpyc_k = caxpyc_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cscal_dispatch_t openblas_cscal_dispatchTS = { .cscal_k = cscal_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cswap_dispatch_t openblas_cswap_dispatchTS = { .cswap_k = cswap_kTS, +}; +#endif +#if (BUILD_COMPLEX==1) +const openblas_cgemv_dispatch_t openblas_cgemv_dispatchTS = { .cgemv_n = cgemv_nTS, .cgemv_t = cgemv_tTS, .cgemv_r = cgemv_rTS, @@ -823,20 +1516,51 @@ gotoblas_t TABLE_NAME = { .cgemv_u = cgemv_uTS, .cgemv_s = cgemv_sTS, .cgemv_d = cgemv_dTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_cgeru_dispatch_t openblas_cgeru_dispatchTS = { .cgeru_k = cgeru_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cgerc_dispatch_t openblas_cgerc_dispatchTS = { .cgerc_k = cgerc_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cgerv_dispatch_t openblas_cgerv_dispatchTS = { .cgerv_k = cgerv_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_cgerd_dispatch_t openblas_cgerd_dispatchTS = { .cgerd_k = cgerd_kTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_csymv_dispatch_t openblas_csymv_dispatchTS = { .csymv_L = csymv_LTS, .csymv_U = csymv_UTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_chemv_dispatch_t openblas_chemv_dispatchTS = { .chemv_L = chemv_LTS, .chemv_U = chemv_UTS, .chemv_M = chemv_MTS, .chemv_V = chemv_VTS, +}; #endif -#if (BUILD_COMPLEX) + +#if (BUILD_COMPLEX==1) +const openblas_cgemm_dispatch_t openblas_cgemm_dispatchTS = { #ifdef ARCH_ARM64 #ifdef HAVE_SME .sme_cgemm_kernel = sme_cgemm_kernelTS, @@ -849,14 +1573,6 @@ gotoblas_t TABLE_NAME = { .cgemm_kernel_r = cgemm_kernel_rTS, .cgemm_kernel_b = cgemm_kernel_bTS, .cgemm_beta = cgemm_betaTS, - .csymm_kernel_n = csymm_kernel_nTS, - .csymm_kernel_l = csymm_kernel_lTS, - .csymm_kernel_r = csymm_kernel_rTS, - .csymm_kernel_b = csymm_kernel_bTS, - .ctrmm_gemm_kernel_n = ctrmm_gemm_kernel_nTS, - .ctrmm_gemm_kernel_l = ctrmm_gemm_kernel_lTS, - .ctrmm_gemm_kernel_r = ctrmm_gemm_kernel_rTS, - .ctrmm_gemm_kernel_b = ctrmm_gemm_kernel_bTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N .cgemm_incopy = cgemm_incopyTS, .cgemm_itcopy = cgemm_itcopyTS, @@ -866,7 +1582,6 @@ gotoblas_t TABLE_NAME = { #endif .cgemm_oncopy = cgemm_oncopyTS, .cgemm_otcopy = cgemm_otcopyTS, - #ifdef SMALL_MATRIX_OPT .cgemm_small_matrix_permit = cgemm_small_matrix_permitTS, .cgemm_small_kernel_nn = cgemm_small_kernel_nnTS, @@ -902,47 +1617,35 @@ gotoblas_t TABLE_NAME = { .cgemm_small_kernel_b0_cr = cgemm_small_kernel_b0_crTS, .cgemm_small_kernel_b0_cc = cgemm_small_kernel_b0_ccTS, #endif +}; +#endif - .ctrsm_kernel_LN = ctrsm_kernel_LNTS, - .ctrsm_kernel_LT = ctrsm_kernel_LTTS, - .ctrsm_kernel_LR = ctrsm_kernel_LRTS, - .ctrsm_kernel_LC = ctrsm_kernel_LCTS, - .ctrsm_kernel_RN = ctrsm_kernel_RNTS, - .ctrsm_kernel_RT = ctrsm_kernel_RTTS, - .ctrsm_kernel_RR = ctrsm_kernel_RRTS, - .ctrsm_kernel_RC = ctrsm_kernel_RCTS, - +#if (BUILD_COMPLEX==1) +const openblas_csymm_dispatch_t openblas_csymm_dispatchTS = { + .csymm_kernel_n = csymm_kernel_nTS, + .csymm_kernel_l = csymm_kernel_lTS, + .csymm_kernel_r = csymm_kernel_rTS, + .csymm_kernel_b = csymm_kernel_bTS, + .csymm_incopy = csymm_incopyTS, + .csymm_itcopy = csymm_itcopyTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - .ctrsm_iunucopy = ctrsm_iunucopyTS, - .ctrsm_iunncopy = ctrsm_iunncopyTS, - .ctrsm_iutucopy = ctrsm_iutucopyTS, - .ctrsm_iutncopy = ctrsm_iutncopyTS, - .ctrsm_ilnucopy = ctrsm_ilnucopyTS, - .ctrsm_ilnncopy = ctrsm_ilnncopyTS, - .ctrsm_iltucopy = ctrsm_iltucopyTS, - .ctrsm_iltncopy = ctrsm_iltncopyTS, + .csymm_iutcopy = csymm_iutcopyTS, + .csymm_iltcopy = csymm_iltcopyTS, #else - .ctrsm_iunucopy = ctrsm_ounucopyTS, - .ctrsm_iunncopy = ctrsm_ounncopyTS, - .ctrsm_iutucopy = ctrsm_outucopyTS, - .ctrsm_iutncopy = ctrsm_outncopyTS, - .ctrsm_ilnucopy = ctrsm_olnucopyTS, - .ctrsm_ilnncopy = ctrsm_olnncopyTS, - .ctrsm_iltucopy = ctrsm_oltucopyTS, - .ctrsm_iltncopy = ctrsm_oltncopyTS, -#endif - .ctrsm_ounucopy = ctrsm_ounucopyTS, - .ctrsm_ounncopy = ctrsm_ounncopyTS, - .ctrsm_outucopy = ctrsm_outucopyTS, - .ctrsm_outncopy = ctrsm_outncopyTS, - .ctrsm_olnucopy = ctrsm_olnucopyTS, - .ctrsm_olnncopy = ctrsm_olnncopyTS, - .ctrsm_oltucopy = ctrsm_oltucopyTS, - .ctrsm_oltncopy = ctrsm_oltncopyTS, + .csymm_iutcopy = csymm_outcopyTS, + .csymm_iltcopy = csymm_oltcopyTS, #endif + .csymm_outcopy = csymm_outcopyTS, + .csymm_oltcopy = csymm_oltcopyTS, +}; #endif -#if (BUILD_COMPLEX) +#if (BUILD_COMPLEX==1) +const openblas_ctrmm_dispatch_t openblas_ctrmm_dispatchTS = { + .ctrmm_gemm_kernel_n = ctrmm_gemm_kernel_nTS, + .ctrmm_gemm_kernel_l = ctrmm_gemm_kernel_lTS, + .ctrmm_gemm_kernel_r = ctrmm_gemm_kernel_rTS, + .ctrmm_gemm_kernel_b = ctrmm_gemm_kernel_bTS, .ctrmm_kernel_RN = ctrmm_kernel_RNTS, .ctrmm_kernel_RT = ctrmm_kernel_RTTS, .ctrmm_kernel_RR = ctrmm_kernel_RRTS, @@ -951,7 +1654,6 @@ gotoblas_t TABLE_NAME = { .ctrmm_kernel_LT = ctrmm_kernel_LTTS, .ctrmm_kernel_LR = ctrmm_kernel_LRTS, .ctrmm_kernel_LC = ctrmm_kernel_LCTS, - #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N .ctrmm_iunucopy = ctrmm_iunucopyTS, .ctrmm_iunncopy = ctrmm_iunncopyTS, @@ -971,8 +1673,6 @@ gotoblas_t TABLE_NAME = { .ctrmm_iltucopy = ctrmm_oltucopyTS, .ctrmm_iltncopy = ctrmm_oltncopyTS, #endif - .csymm_incopy = csymm_incopyTS, - .csymm_itcopy = csymm_itcopyTS, .ctrmm_incopy = ctrmm_incopyTS, .ctrmm_itcopy = ctrmm_itcopyTS, .ctrmm_ounucopy = ctrmm_ounucopyTS, @@ -983,16 +1683,51 @@ gotoblas_t TABLE_NAME = { .ctrmm_olnncopy = ctrmm_olnncopyTS, .ctrmm_oltucopy = ctrmm_oltucopyTS, .ctrmm_oltncopy = ctrmm_oltncopyTS, +}; +#endif +#if (BUILD_COMPLEX==1) +const openblas_ctrsm_dispatch_t openblas_ctrsm_dispatchTS = { + .ctrsm_kernel_LN = ctrsm_kernel_LNTS, + .ctrsm_kernel_LT = ctrsm_kernel_LTTS, + .ctrsm_kernel_LR = ctrsm_kernel_LRTS, + .ctrsm_kernel_LC = ctrsm_kernel_LCTS, + .ctrsm_kernel_RN = ctrsm_kernel_RNTS, + .ctrsm_kernel_RT = ctrsm_kernel_RTTS, + .ctrsm_kernel_RR = ctrsm_kernel_RRTS, + .ctrsm_kernel_RC = ctrsm_kernel_RCTS, #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N - .csymm_iutcopy = csymm_iutcopyTS, - .csymm_iltcopy = csymm_iltcopyTS, + .ctrsm_iunucopy = ctrsm_iunucopyTS, + .ctrsm_iunncopy = ctrsm_iunncopyTS, + .ctrsm_iutucopy = ctrsm_iutucopyTS, + .ctrsm_iutncopy = ctrsm_iutncopyTS, + .ctrsm_ilnucopy = ctrsm_ilnucopyTS, + .ctrsm_ilnncopy = ctrsm_ilnncopyTS, + .ctrsm_iltucopy = ctrsm_iltucopyTS, + .ctrsm_iltncopy = ctrsm_iltncopyTS, #else - .csymm_iutcopy = csymm_outcopyTS, - .csymm_iltcopy = csymm_oltcopyTS, + .ctrsm_iunucopy = ctrsm_ounucopyTS, + .ctrsm_iunncopy = ctrsm_ounncopyTS, + .ctrsm_iutucopy = ctrsm_outucopyTS, + .ctrsm_iutncopy = ctrsm_outncopyTS, + .ctrsm_ilnucopy = ctrsm_olnucopyTS, + .ctrsm_ilnncopy = ctrsm_olnncopyTS, + .ctrsm_iltucopy = ctrsm_oltucopyTS, + .ctrsm_iltncopy = ctrsm_oltncopyTS, #endif - .csymm_outcopy = csymm_outcopyTS, - .csymm_oltcopy = csymm_oltcopyTS, + .ctrsm_ounucopy = ctrsm_ounucopyTS, + .ctrsm_ounncopy = ctrsm_ounncopyTS, + .ctrsm_outucopy = ctrsm_outucopyTS, + .ctrsm_outncopy = ctrsm_outncopyTS, + .ctrsm_olnucopy = ctrsm_olnucopyTS, + .ctrsm_olnncopy = ctrsm_olnncopyTS, + .ctrsm_oltucopy = ctrsm_oltucopyTS, + .ctrsm_oltncopy = ctrsm_oltncopyTS, +}; +#endif + +#if (BUILD_COMPLEX==1) +const openblas_chemm_dispatch_t openblas_chemm_dispatchTS = { #if CGEMM_DEFAULT_UNROLL_M != CGEMM_DEFAULT_UNROLL_N .chemm_iutcopy = chemm_iutcopyTS, .chemm_iltcopy = chemm_iltcopyTS, @@ -1002,25 +1737,13 @@ gotoblas_t TABLE_NAME = { #endif .chemm_outcopy = chemm_outcopyTS, .chemm_oltcopy = chemm_oltcopyTS, - - .cgemm3m_p = 0, - .cgemm3m_q = 0, - .cgemm3m_r = 0, - -#if (USE_GEMM3M) -#ifdef CGEMM3M_DEFAULT_UNROLL_M - .cgemm3m_unroll_m = CGEMM3M_DEFAULT_UNROLL_M, - .cgemm3m_unroll_n = CGEMM3M_DEFAULT_UNROLL_N, - .cgemm3m_unroll_mn = MAX(CGEMM3M_DEFAULT_UNROLL_M, CGEMM3M_DEFAULT_UNROLL_N), -#else - .cgemm3m_unroll_m = SGEMM_DEFAULT_UNROLL_M, - .cgemm3m_unroll_n = SGEMM_DEFAULT_UNROLL_N, - .cgemm3m_unroll_mn = MAX(SGEMM_DEFAULT_UNROLL_M, SGEMM_DEFAULT_UNROLL_N), +}; #endif - +#if (BUILD_COMPLEX==1) +const openblas_cgemm3m_dispatch_t openblas_cgemm3m_dispatchTS = { +#if (USE_GEMM3M) .cgemm3m_kernel = cgemm3m_kernelTS, - .cgemm3m_incopyb = cgemm3m_incopybTS, .cgemm3m_incopyr = cgemm3m_incopyrTS, .cgemm3m_incopyi = cgemm3m_incopyiTS, @@ -1033,40 +1756,8 @@ gotoblas_t TABLE_NAME = { .cgemm3m_otcopyb = cgemm3m_otcopybTS, .cgemm3m_otcopyr = cgemm3m_otcopyrTS, .cgemm3m_otcopyi = cgemm3m_otcopyiTS, - - .csymm3m_iucopyb = csymm3m_iucopybTS, - .csymm3m_ilcopyb = csymm3m_ilcopybTS, - .csymm3m_iucopyr = csymm3m_iucopyrTS, - .csymm3m_ilcopyr = csymm3m_ilcopyrTS, - .csymm3m_iucopyi = csymm3m_iucopyiTS, - .csymm3m_ilcopyi = csymm3m_ilcopyiTS, - .csymm3m_oucopyb = csymm3m_oucopybTS, - .csymm3m_olcopyb = csymm3m_olcopybTS, - .csymm3m_oucopyr = csymm3m_oucopyrTS, - .csymm3m_olcopyr = csymm3m_olcopyrTS, - .csymm3m_oucopyi = csymm3m_oucopyiTS, - .csymm3m_olcopyi = csymm3m_olcopyiTS, - - .chemm3m_iucopyb = chemm3m_iucopybTS, - .chemm3m_ilcopyb = chemm3m_ilcopybTS, - .chemm3m_iucopyr = chemm3m_iucopyrTS, - .chemm3m_ilcopyr = chemm3m_ilcopyrTS, - .chemm3m_iucopyi = chemm3m_iucopyiTS, - .chemm3m_ilcopyi = chemm3m_ilcopyiTS, - - .chemm3m_oucopyb = chemm3m_oucopybTS, - .chemm3m_olcopyb = chemm3m_olcopybTS, - .chemm3m_oucopyr = chemm3m_oucopyrTS, - .chemm3m_olcopyr = chemm3m_olcopyrTS, - .chemm3m_oucopyi = chemm3m_oucopyiTS, - .chemm3m_olcopyi = chemm3m_olcopyiTS, #else - .cgemm3m_unroll_m = 0, - .cgemm3m_unroll_n = 0, - .cgemm3m_unroll_mn = 0, - .cgemm3m_kernel = NULL, - .cgemm3m_incopyb = NULL, .cgemm3m_incopyr = NULL, .cgemm3m_incopyi = NULL, @@ -1079,7 +1770,26 @@ gotoblas_t TABLE_NAME = { .cgemm3m_otcopyb = NULL, .cgemm3m_otcopyr = NULL, .cgemm3m_otcopyi = NULL, +#endif +}; +#endif +#if (BUILD_COMPLEX==1) +const openblas_csymm3m_dispatch_t openblas_csymm3m_dispatchTS = { +#if (USE_GEMM3M) + .csymm3m_iucopyb = csymm3m_iucopybTS, + .csymm3m_ilcopyb = csymm3m_ilcopybTS, + .csymm3m_iucopyr = csymm3m_iucopyrTS, + .csymm3m_ilcopyr = csymm3m_ilcopyrTS, + .csymm3m_iucopyi = csymm3m_iucopyiTS, + .csymm3m_ilcopyi = csymm3m_ilcopyiTS, + .csymm3m_oucopyb = csymm3m_oucopybTS, + .csymm3m_olcopyb = csymm3m_olcopybTS, + .csymm3m_oucopyr = csymm3m_oucopyrTS, + .csymm3m_olcopyr = csymm3m_olcopyrTS, + .csymm3m_oucopyi = csymm3m_oucopyiTS, + .csymm3m_olcopyi = csymm3m_olcopyiTS, +#else .csymm3m_iucopyb = NULL, .csymm3m_ilcopyb = NULL, .csymm3m_iucopyr = NULL, @@ -1092,14 +1802,32 @@ gotoblas_t TABLE_NAME = { .csymm3m_olcopyr = NULL, .csymm3m_oucopyi = NULL, .csymm3m_olcopyi = NULL, +#endif +}; +#endif +#if (BUILD_COMPLEX==1) +const openblas_chemm3m_dispatch_t openblas_chemm3m_dispatchTS = { +#if (USE_GEMM3M) + .chemm3m_iucopyb = chemm3m_iucopybTS, + .chemm3m_ilcopyb = chemm3m_ilcopybTS, + .chemm3m_iucopyr = chemm3m_iucopyrTS, + .chemm3m_ilcopyr = chemm3m_ilcopyrTS, + .chemm3m_iucopyi = chemm3m_iucopyiTS, + .chemm3m_ilcopyi = chemm3m_ilcopyiTS, + .chemm3m_oucopyb = chemm3m_oucopybTS, + .chemm3m_olcopyb = chemm3m_olcopybTS, + .chemm3m_oucopyr = chemm3m_oucopyrTS, + .chemm3m_olcopyr = chemm3m_olcopyrTS, + .chemm3m_oucopyi = chemm3m_oucopyiTS, + .chemm3m_olcopyi = chemm3m_olcopyiTS, +#else .chemm3m_iucopyb = NULL, .chemm3m_ilcopyb = NULL, .chemm3m_iucopyr = NULL, .chemm3m_ilcopyr = NULL, .chemm3m_iucopyi = NULL, .chemm3m_ilcopyi = NULL, - .chemm3m_oucopyb = NULL, .chemm3m_olcopyb = NULL, .chemm3m_oucopyr = NULL, @@ -1107,48 +1835,121 @@ gotoblas_t TABLE_NAME = { .chemm3m_oucopyi = NULL, .chemm3m_olcopyi = NULL, #endif +}; #endif -#if (BUILD_COMPLEX) +#if (BUILD_COMPLEX==1) +const openblas_cneg_dispatch_t openblas_cneg_dispatchTS = { #ifndef NO_LAPACK .cneg_tcopy = cneg_tcopyTS, - - .claswp_ncopy = claswp_ncopyTS, #else .cneg_tcopy = NULL, - .claswp_ncopy = NULL, #endif - +}; #endif -#if BUILD_COMPLEX16 == 1 - .zgemm_p = 0, - .zgemm_q = 0, - .zgemm_r = 0, - .zgemm_unroll_m = ZGEMM_DEFAULT_UNROLL_M, - .zgemm_unroll_n = ZGEMM_DEFAULT_UNROLL_N, -#ifdef ZGEMM_DEFAULT_UNROLL_MN - .zgemm_unroll_mn = ZGEMM_DEFAULT_UNROLL_MN, +#if (BUILD_COMPLEX==1) +const openblas_claswp_dispatch_t openblas_claswp_dispatchTS = { +#ifndef NO_LAPACK + .claswp_ncopy = claswp_ncopyTS, #else - .zgemm_unroll_mn = MAX(ZGEMM_DEFAULT_UNROLL_M, ZGEMM_DEFAULT_UNROLL_N), + .claswp_ncopy = NULL, +#endif +}; #endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zamax_dispatch_t openblas_zamax_dispatchTS = { .zamax_k = zamax_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zamin_dispatch_t openblas_zamin_dispatchTS = { .zamin_k = zamin_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_izamax_dispatch_t openblas_izamax_dispatchTS = { .izamax_k = izamax_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_izamin_dispatch_t openblas_izamin_dispatchTS = { .izamin_k = izamin_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_znrm2_dispatch_t openblas_znrm2_dispatchTS = { .znrm2_k = znrm2_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zasum_dispatch_t openblas_zasum_dispatchTS = { .zasum_k = zasum_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zsum_dispatch_t openblas_zsum_dispatchTS = { .zsum_k = zsum_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zcopy_dispatch_t openblas_zcopy_dispatchTS = { .zcopy_k = zcopy_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdotu_dispatch_t openblas_zdotu_dispatchTS = { .zdotu_k = zdotu_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdotc_dispatch_t openblas_zdotc_dispatchTS = { .zdotc_k = zdotc_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zdrot_dispatch_t openblas_zdrot_dispatchTS = { .zdrot_k = zdrot_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zaxpy_dispatch_t openblas_zaxpy_dispatchTS = { .zaxpy_k = zaxpy_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zaxpyc_dispatch_t openblas_zaxpyc_dispatchTS = { .zaxpyc_k = zaxpyc_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zscal_dispatch_t openblas_zscal_dispatchTS = { .zscal_k = zscal_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zswap_dispatch_t openblas_zswap_dispatchTS = { .zswap_k = zswap_kTS, +}; +#endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemv_dispatch_t openblas_zgemv_dispatchTS = { .zgemv_n = zgemv_nTS, .zgemv_t = zgemv_tTS, .zgemv_r = zgemv_rTS, @@ -1157,16 +1958,51 @@ gotoblas_t TABLE_NAME = { .zgemv_u = zgemv_uTS, .zgemv_s = zgemv_sTS, .zgemv_d = zgemv_dTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgeru_dispatch_t openblas_zgeru_dispatchTS = { .zgeru_k = zgeru_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerc_dispatch_t openblas_zgerc_dispatchTS = { .zgerc_k = zgerc_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerv_dispatch_t openblas_zgerv_dispatchTS = { .zgerv_k = zgerv_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgerd_dispatch_t openblas_zgerd_dispatchTS = { .zgerd_k = zgerd_kTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymv_dispatch_t openblas_zsymv_dispatchTS = { .zsymv_L = zsymv_LTS, .zsymv_U = zsymv_UTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemv_dispatch_t openblas_zhemv_dispatchTS = { .zhemv_L = zhemv_LTS, .zhemv_U = zhemv_UTS, .zhemv_M = zhemv_MTS, .zhemv_V = zhemv_VTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemm_dispatch_t openblas_zgemm_dispatchTS = { #ifdef ARCH_ARM64 #ifdef HAVE_SME .sme_zgemm_kernel = sme_zgemm_kernelTS, @@ -1179,15 +2015,6 @@ gotoblas_t TABLE_NAME = { .zgemm_kernel_r = zgemm_kernel_rTS, .zgemm_kernel_b = zgemm_kernel_bTS, .zgemm_beta = zgemm_betaTS, - .zsymm_kernel_n = zsymm_kernel_nTS, - .zsymm_kernel_l = zsymm_kernel_lTS, - .zsymm_kernel_r = zsymm_kernel_rTS, - .zsymm_kernel_b = zsymm_kernel_bTS, - .ztrmm_gemm_kernel_n = ztrmm_gemm_kernel_nTS, - .ztrmm_gemm_kernel_l = ztrmm_gemm_kernel_lTS, - .ztrmm_gemm_kernel_r = ztrmm_gemm_kernel_rTS, - .ztrmm_gemm_kernel_b = ztrmm_gemm_kernel_bTS, - #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N .zgemm_incopy = zgemm_incopyTS, .zgemm_itcopy = zgemm_itcopyTS, @@ -1197,7 +2024,6 @@ gotoblas_t TABLE_NAME = { #endif .zgemm_oncopy = zgemm_oncopyTS, .zgemm_otcopy = zgemm_otcopyTS, - #ifdef SMALL_MATRIX_OPT .zgemm_small_matrix_permit = zgemm_small_matrix_permitTS, .zgemm_small_kernel_nn = zgemm_small_kernel_nnTS, @@ -1233,44 +2059,35 @@ gotoblas_t TABLE_NAME = { .zgemm_small_kernel_b0_cr = zgemm_small_kernel_b0_crTS, .zgemm_small_kernel_b0_cc = zgemm_small_kernel_b0_ccTS, #endif +}; +#endif - .ztrsm_kernel_LN = ztrsm_kernel_LNTS, - .ztrsm_kernel_LT = ztrsm_kernel_LTTS, - .ztrsm_kernel_LR = ztrsm_kernel_LRTS, - .ztrsm_kernel_LC = ztrsm_kernel_LCTS, - .ztrsm_kernel_RN = ztrsm_kernel_RNTS, - .ztrsm_kernel_RT = ztrsm_kernel_RTTS, - .ztrsm_kernel_RR = ztrsm_kernel_RRTS, - .ztrsm_kernel_RC = ztrsm_kernel_RCTS, - +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymm_dispatch_t openblas_zsymm_dispatchTS = { + .zsymm_kernel_n = zsymm_kernel_nTS, + .zsymm_kernel_l = zsymm_kernel_lTS, + .zsymm_kernel_r = zsymm_kernel_rTS, + .zsymm_kernel_b = zsymm_kernel_bTS, + .zsymm_incopy = zsymm_incopyTS, + .zsymm_itcopy = zsymm_itcopyTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - .ztrsm_iunucopy = ztrsm_iunucopyTS, - .ztrsm_iunncopy = ztrsm_iunncopyTS, - .ztrsm_iutucopy = ztrsm_iutucopyTS, - .ztrsm_iutncopy = ztrsm_iutncopyTS, - .ztrsm_ilnucopy = ztrsm_ilnucopyTS, - .ztrsm_ilnncopy = ztrsm_ilnncopyTS, - .ztrsm_iltucopy = ztrsm_iltucopyTS, - .ztrsm_iltncopy = ztrsm_iltncopyTS, + .zsymm_iutcopy = zsymm_iutcopyTS, + .zsymm_iltcopy = zsymm_iltcopyTS, #else - .ztrsm_iunucopy = ztrsm_ounucopyTS, - .ztrsm_iunncopy = ztrsm_ounncopyTS, - .ztrsm_iutucopy = ztrsm_outucopyTS, - .ztrsm_iutncopy = ztrsm_outncopyTS, - .ztrsm_ilnucopy = ztrsm_olnucopyTS, - .ztrsm_ilnncopy = ztrsm_olnncopyTS, - .ztrsm_iltucopy = ztrsm_oltucopyTS, - .ztrsm_iltncopy = ztrsm_oltncopyTS, + .zsymm_iutcopy = zsymm_outcopyTS, + .zsymm_iltcopy = zsymm_oltcopyTS, +#endif + .zsymm_outcopy = zsymm_outcopyTS, + .zsymm_oltcopy = zsymm_oltcopyTS, +}; #endif - .ztrsm_ounucopy = ztrsm_ounucopyTS, - .ztrsm_ounncopy = ztrsm_ounncopyTS, - .ztrsm_outucopy = ztrsm_outucopyTS, - .ztrsm_outncopy = ztrsm_outncopyTS, - .ztrsm_olnucopy = ztrsm_olnucopyTS, - .ztrsm_olnncopy = ztrsm_olnncopyTS, - .ztrsm_oltucopy = ztrsm_oltucopyTS, - .ztrsm_oltncopy = ztrsm_oltncopyTS, +#if (BUILD_COMPLEX16 == 1) +const openblas_ztrmm_dispatch_t openblas_ztrmm_dispatchTS = { + .ztrmm_gemm_kernel_n = ztrmm_gemm_kernel_nTS, + .ztrmm_gemm_kernel_l = ztrmm_gemm_kernel_lTS, + .ztrmm_gemm_kernel_r = ztrmm_gemm_kernel_rTS, + .ztrmm_gemm_kernel_b = ztrmm_gemm_kernel_bTS, .ztrmm_kernel_RN = ztrmm_kernel_RNTS, .ztrmm_kernel_RT = ztrmm_kernel_RTTS, .ztrmm_kernel_RR = ztrmm_kernel_RRTS, @@ -1279,7 +2096,6 @@ gotoblas_t TABLE_NAME = { .ztrmm_kernel_LT = ztrmm_kernel_LTTS, .ztrmm_kernel_LR = ztrmm_kernel_LRTS, .ztrmm_kernel_LC = ztrmm_kernel_LCTS, - #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N .ztrmm_iunucopy = ztrmm_iunucopyTS, .ztrmm_iunncopy = ztrmm_iunncopyTS, @@ -1299,8 +2115,6 @@ gotoblas_t TABLE_NAME = { .ztrmm_iltucopy = ztrmm_oltucopyTS, .ztrmm_iltncopy = ztrmm_oltncopyTS, #endif - .zsymm_incopy = zsymm_incopyTS, - .zsymm_itcopy = zsymm_itcopyTS, .ztrmm_incopy = ztrmm_incopyTS, .ztrmm_itcopy = ztrmm_itcopyTS, .ztrmm_ounucopy = ztrmm_ounucopyTS, @@ -1311,16 +2125,51 @@ gotoblas_t TABLE_NAME = { .ztrmm_olnncopy = ztrmm_olnncopyTS, .ztrmm_oltucopy = ztrmm_oltucopyTS, .ztrmm_oltncopy = ztrmm_oltncopyTS, +}; +#endif +#if (BUILD_COMPLEX16 == 1) +const openblas_ztrsm_dispatch_t openblas_ztrsm_dispatchTS = { + .ztrsm_kernel_LN = ztrsm_kernel_LNTS, + .ztrsm_kernel_LT = ztrsm_kernel_LTTS, + .ztrsm_kernel_LR = ztrsm_kernel_LRTS, + .ztrsm_kernel_LC = ztrsm_kernel_LCTS, + .ztrsm_kernel_RN = ztrsm_kernel_RNTS, + .ztrsm_kernel_RT = ztrsm_kernel_RTTS, + .ztrsm_kernel_RR = ztrsm_kernel_RRTS, + .ztrsm_kernel_RC = ztrsm_kernel_RCTS, #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N - .zsymm_iutcopy = zsymm_iutcopyTS, - .zsymm_iltcopy = zsymm_iltcopyTS, + .ztrsm_iunucopy = ztrsm_iunucopyTS, + .ztrsm_iunncopy = ztrsm_iunncopyTS, + .ztrsm_iutucopy = ztrsm_iutucopyTS, + .ztrsm_iutncopy = ztrsm_iutncopyTS, + .ztrsm_ilnucopy = ztrsm_ilnucopyTS, + .ztrsm_ilnncopy = ztrsm_ilnncopyTS, + .ztrsm_iltucopy = ztrsm_iltucopyTS, + .ztrsm_iltncopy = ztrsm_iltncopyTS, #else - .zsymm_iutcopy = zsymm_outcopyTS, - .zsymm_iltcopy = zsymm_oltcopyTS, + .ztrsm_iunucopy = ztrsm_ounucopyTS, + .ztrsm_iunncopy = ztrsm_ounncopyTS, + .ztrsm_iutucopy = ztrsm_outucopyTS, + .ztrsm_iutncopy = ztrsm_outncopyTS, + .ztrsm_ilnucopy = ztrsm_olnucopyTS, + .ztrsm_ilnncopy = ztrsm_olnncopyTS, + .ztrsm_iltucopy = ztrsm_oltucopyTS, + .ztrsm_iltncopy = ztrsm_oltncopyTS, #endif - .zsymm_outcopy = zsymm_outcopyTS, - .zsymm_oltcopy = zsymm_oltcopyTS, + .ztrsm_ounucopy = ztrsm_ounucopyTS, + .ztrsm_ounncopy = ztrsm_ounncopyTS, + .ztrsm_outucopy = ztrsm_outucopyTS, + .ztrsm_outncopy = ztrsm_outncopyTS, + .ztrsm_olnucopy = ztrsm_olnucopyTS, + .ztrsm_olnncopy = ztrsm_olnncopyTS, + .ztrsm_oltucopy = ztrsm_oltucopyTS, + .ztrsm_oltncopy = ztrsm_oltncopyTS, +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemm_dispatch_t openblas_zhemm_dispatchTS = { #if ZGEMM_DEFAULT_UNROLL_M != ZGEMM_DEFAULT_UNROLL_N .zhemm_iutcopy = zhemm_iutcopyTS, .zhemm_iltcopy = zhemm_iltcopyTS, @@ -1330,24 +2179,13 @@ gotoblas_t TABLE_NAME = { #endif .zhemm_outcopy = zhemm_outcopyTS, .zhemm_oltcopy = zhemm_oltcopyTS, - - .zgemm3m_p = 0, - .zgemm3m_q = 0, - .zgemm3m_r = 0, -#if (USE_GEMM3M) -#ifdef ZGEMM3M_DEFAULT_UNROLL_M - .zgemm3m_unroll_m = ZGEMM3M_DEFAULT_UNROLL_M, - .zgemm3m_unroll_n = ZGEMM3M_DEFAULT_UNROLL_N, - .zgemm3m_unroll_mn = MAX(ZGEMM3M_DEFAULT_UNROLL_M, ZGEMM3M_DEFAULT_UNROLL_N), -#else - .zgemm3m_unroll_m = DGEMM_DEFAULT_UNROLL_M, - .zgemm3m_unroll_n = DGEMM_DEFAULT_UNROLL_N, - .zgemm3m_unroll_mn = MAX(DGEMM_DEFAULT_UNROLL_M, DGEMM_DEFAULT_UNROLL_N), +}; #endif - +#if (BUILD_COMPLEX16 == 1) +const openblas_zgemm3m_dispatch_t openblas_zgemm3m_dispatchTS = { +#if (USE_GEMM3M) .zgemm3m_kernel = zgemm3m_kernelTS, - .zgemm3m_incopyb = zgemm3m_incopybTS, .zgemm3m_incopyr = zgemm3m_incopyrTS, .zgemm3m_incopyi = zgemm3m_incopyiTS, @@ -1360,40 +2198,8 @@ gotoblas_t TABLE_NAME = { .zgemm3m_otcopyb = zgemm3m_otcopybTS, .zgemm3m_otcopyr = zgemm3m_otcopyrTS, .zgemm3m_otcopyi = zgemm3m_otcopyiTS, - - .zsymm3m_iucopyb = zsymm3m_iucopybTS, - .zsymm3m_ilcopyb = zsymm3m_ilcopybTS, - .zsymm3m_iucopyr = zsymm3m_iucopyrTS, - .zsymm3m_ilcopyr = zsymm3m_ilcopyrTS, - .zsymm3m_iucopyi = zsymm3m_iucopyiTS, - .zsymm3m_ilcopyi = zsymm3m_ilcopyiTS, - .zsymm3m_oucopyb = zsymm3m_oucopybTS, - .zsymm3m_olcopyb = zsymm3m_olcopybTS, - .zsymm3m_oucopyr = zsymm3m_oucopyrTS, - .zsymm3m_olcopyr = zsymm3m_olcopyrTS, - .zsymm3m_oucopyi = zsymm3m_oucopyiTS, - .zsymm3m_olcopyi = zsymm3m_olcopyiTS, - - .zhemm3m_iucopyb = zhemm3m_iucopybTS, - .zhemm3m_ilcopyb = zhemm3m_ilcopybTS, - .zhemm3m_iucopyr = zhemm3m_iucopyrTS, - .zhemm3m_ilcopyr = zhemm3m_ilcopyrTS, - .zhemm3m_iucopyi = zhemm3m_iucopyiTS, - .zhemm3m_ilcopyi = zhemm3m_ilcopyiTS, - - .zhemm3m_oucopyb = zhemm3m_oucopybTS, - .zhemm3m_olcopyb = zhemm3m_olcopybTS, - .zhemm3m_oucopyr = zhemm3m_oucopyrTS, - .zhemm3m_olcopyr = zhemm3m_olcopyrTS, - .zhemm3m_oucopyi = zhemm3m_oucopyiTS, - .zhemm3m_olcopyi = zhemm3m_olcopyiTS, #else - .zgemm3m_unroll_m = 0, - .zgemm3m_unroll_n = 0, - .zgemm3m_unroll_mn = 0, - .zgemm3m_kernel = NULL, - .zgemm3m_incopyb = NULL, .zgemm3m_incopyr = NULL, .zgemm3m_incopyi = NULL, @@ -1406,7 +2212,26 @@ gotoblas_t TABLE_NAME = { .zgemm3m_otcopyb = NULL, .zgemm3m_otcopyr = NULL, .zgemm3m_otcopyi = NULL, +#endif +}; +#endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zsymm3m_dispatch_t openblas_zsymm3m_dispatchTS = { +#if (USE_GEMM3M) + .zsymm3m_iucopyb = zsymm3m_iucopybTS, + .zsymm3m_ilcopyb = zsymm3m_ilcopybTS, + .zsymm3m_iucopyr = zsymm3m_iucopyrTS, + .zsymm3m_ilcopyr = zsymm3m_ilcopyrTS, + .zsymm3m_iucopyi = zsymm3m_iucopyiTS, + .zsymm3m_ilcopyi = zsymm3m_ilcopyiTS, + .zsymm3m_oucopyb = zsymm3m_oucopybTS, + .zsymm3m_olcopyb = zsymm3m_olcopybTS, + .zsymm3m_oucopyr = zsymm3m_oucopyrTS, + .zsymm3m_olcopyr = zsymm3m_olcopyrTS, + .zsymm3m_oucopyi = zsymm3m_oucopyiTS, + .zsymm3m_olcopyi = zsymm3m_olcopyiTS, +#else .zsymm3m_iucopyb = NULL, .zsymm3m_ilcopyb = NULL, .zsymm3m_iucopyr = NULL, @@ -1419,14 +2244,32 @@ gotoblas_t TABLE_NAME = { .zsymm3m_olcopyr = NULL, .zsymm3m_oucopyi = NULL, .zsymm3m_olcopyi = NULL, +#endif +}; +#endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zhemm3m_dispatch_t openblas_zhemm3m_dispatchTS = { +#if (USE_GEMM3M) + .zhemm3m_iucopyb = zhemm3m_iucopybTS, + .zhemm3m_ilcopyb = zhemm3m_ilcopybTS, + .zhemm3m_iucopyr = zhemm3m_iucopyrTS, + .zhemm3m_ilcopyr = zhemm3m_ilcopyrTS, + .zhemm3m_iucopyi = zhemm3m_iucopyiTS, + .zhemm3m_ilcopyi = zhemm3m_ilcopyiTS, + .zhemm3m_oucopyb = zhemm3m_oucopybTS, + .zhemm3m_olcopyb = zhemm3m_olcopybTS, + .zhemm3m_oucopyr = zhemm3m_oucopyrTS, + .zhemm3m_olcopyr = zhemm3m_olcopyrTS, + .zhemm3m_oucopyi = zhemm3m_oucopyiTS, + .zhemm3m_olcopyi = zhemm3m_olcopyiTS, +#else .zhemm3m_iucopyb = NULL, .zhemm3m_ilcopyb = NULL, .zhemm3m_iucopyr = NULL, .zhemm3m_ilcopyr = NULL, .zhemm3m_iucopyi = NULL, .zhemm3m_ilcopyi = NULL, - .zhemm3m_oucopyb = NULL, .zhemm3m_olcopyb = NULL, .zhemm3m_oucopyr = NULL, @@ -1434,42 +2277,121 @@ gotoblas_t TABLE_NAME = { .zhemm3m_oucopyi = NULL, .zhemm3m_olcopyi = NULL, #endif +}; +#endif +#if (BUILD_COMPLEX16 == 1) +const openblas_zneg_dispatch_t openblas_zneg_dispatchTS = { #ifndef NO_LAPACK .zneg_tcopy = zneg_tcopyTS, - .zlaswp_ncopy = zlaswp_ncopyTS, #else .zneg_tcopy = NULL, +#endif +}; +#endif + +#if (BUILD_COMPLEX16 == 1) +const openblas_zlaswp_dispatch_t openblas_zlaswp_dispatchTS = { +#ifndef NO_LAPACK + .zlaswp_ncopy = zlaswp_ncopyTS, +#else .zlaswp_ncopy = NULL, #endif +}; +#endif + +#ifdef EXPRECISION +const openblas_xamax_dispatch_t openblas_xamax_dispatchTS = { + .xamax_k = xamax_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xamin_dispatch_t openblas_xamin_dispatchTS = { + .xamin_k = xamin_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_ixamax_dispatch_t openblas_ixamax_dispatchTS = { + .ixamax_k = ixamax_kTS, +}; +#endif +#ifdef EXPRECISION +const openblas_ixamin_dispatch_t openblas_ixamin_dispatchTS = { + .ixamin_k = ixamin_kTS, +}; #endif #ifdef EXPRECISION - - .xgemm_p = 0, - .xgemm_q = 0, - .xgemm_r = 0, - .xgemm_unroll_m = XGEMM_DEFAULT_UNROLL_M, - .xgemm_unroll_n = XGEMM_DEFAULT_UNROLL_N, - .xgemm_unroll_mn = MAX(XGEMM_DEFAULT_UNROLL_M, XGEMM_DEFAULT_UNROLL_N), - - .xamax_k = xamax_kTS, - .xamin_k = xamin_kTS, - .ixamax_k = ixamax_kTS, - .ixamin_k = ixamin_kTS, +const openblas_xnrm2_dispatch_t openblas_xnrm2_dispatchTS = { .xnrm2_k = xnrm2_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xasum_dispatch_t openblas_xasum_dispatchTS = { .xasum_k = xasum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xsum_dispatch_t openblas_xsum_dispatchTS = { .xsum_k = xsum_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xcopy_dispatch_t openblas_xcopy_dispatchTS = { .xcopy_k = xcopy_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xdotu_dispatch_t openblas_xdotu_dispatchTS = { .xdotu_k = xdotu_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xdotc_dispatch_t openblas_xdotc_dispatchTS = { .xdotc_k = xdotc_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xqrot_dispatch_t openblas_xqrot_dispatchTS = { .xqrot_k = xqrot_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xaxpy_dispatch_t openblas_xaxpy_dispatchTS = { .xaxpy_k = xaxpy_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xaxpyc_dispatch_t openblas_xaxpyc_dispatchTS = { .xaxpyc_k = xaxpyc_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xscal_dispatch_t openblas_xscal_dispatchTS = { .xscal_k = xscal_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xswap_dispatch_t openblas_xswap_dispatchTS = { .xswap_k = xswap_kTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xgemv_dispatch_t openblas_xgemv_dispatchTS = { .xgemv_n = xgemv_nTS, .xgemv_t = xgemv_tTS, .xgemv_r = xgemv_rTS, @@ -1478,23 +2400,56 @@ gotoblas_t TABLE_NAME = { .xgemv_u = xgemv_uTS, .xgemv_s = xgemv_sTS, .xgemv_d = xgemv_dTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xgeru_dispatch_t openblas_xgeru_dispatchTS = { .xgeru_k = xgeru_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xgerc_dispatch_t openblas_xgerc_dispatchTS = { .xgerc_k = xgerc_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xgerv_dispatch_t openblas_xgerv_dispatchTS = { .xgerv_k = xgerv_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xgerd_dispatch_t openblas_xgerd_dispatchTS = { .xgerd_k = xgerd_kTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xsymv_dispatch_t openblas_xsymv_dispatchTS = { .xsymv_L = xsymv_LTS, .xsymv_U = xsymv_UTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xhemv_dispatch_t openblas_xhemv_dispatchTS = { .xhemv_L = xhemv_LTS, .xhemv_U = xhemv_UTS, .xhemv_M = xhemv_MTS, .xhemv_V = xhemv_VTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xgemm_dispatch_t openblas_xgemm_dispatchTS = { .xgemm_kernel_n = xgemm_kernel_nTS, .xgemm_kernel_l = xgemm_kernel_lTS, .xgemm_kernel_r = xgemm_kernel_rTS, .xgemm_kernel_b = xgemm_kernel_bTS, .xgemm_beta = xgemm_betaTS, - #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N .xgemm_incopy = xgemm_incopyTS, .xgemm_itcopy = xgemm_itcopyTS, @@ -1504,7 +2459,11 @@ gotoblas_t TABLE_NAME = { #endif .xgemm_oncopy = xgemm_oncopyTS, .xgemm_otcopy = xgemm_otcopyTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xtrsm_dispatch_t openblas_xtrsm_dispatchTS = { .xtrsm_kernel_LN = xtrsm_kernel_LNTS, .xtrsm_kernel_LT = xtrsm_kernel_LTTS, .xtrsm_kernel_LR = xtrsm_kernel_LRTS, @@ -1513,7 +2472,6 @@ gotoblas_t TABLE_NAME = { .xtrsm_kernel_RT = xtrsm_kernel_RTTS, .xtrsm_kernel_RR = xtrsm_kernel_RRTS, .xtrsm_kernel_RC = xtrsm_kernel_RCTS, - #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N .xtrsm_iunucopy = xtrsm_iunucopyTS, .xtrsm_iunncopy = xtrsm_iunncopyTS, @@ -1541,7 +2499,11 @@ gotoblas_t TABLE_NAME = { .xtrsm_olnncopy = xtrsm_olnncopyTS, .xtrsm_oltucopy = xtrsm_oltucopyTS, .xtrsm_oltncopy = xtrsm_oltncopyTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xtrmm_dispatch_t openblas_xtrmm_dispatchTS = { .xtrmm_kernel_RN = xtrmm_kernel_RNTS, .xtrmm_kernel_RT = xtrmm_kernel_RTTS, .xtrmm_kernel_RR = xtrmm_kernel_RRTS, @@ -1550,7 +2512,6 @@ gotoblas_t TABLE_NAME = { .xtrmm_kernel_LT = xtrmm_kernel_LTTS, .xtrmm_kernel_LR = xtrmm_kernel_LRTS, .xtrmm_kernel_LC = xtrmm_kernel_LCTS, - #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N .xtrmm_iunucopy = xtrmm_iunucopyTS, .xtrmm_iunncopy = xtrmm_iunncopyTS, @@ -1578,7 +2539,11 @@ gotoblas_t TABLE_NAME = { .xtrmm_olnncopy = xtrmm_olnncopyTS, .xtrmm_oltucopy = xtrmm_oltucopyTS, .xtrmm_oltncopy = xtrmm_oltncopyTS, +}; +#endif +#ifdef EXPRECISION +const openblas_xsymm_dispatch_t openblas_xsymm_dispatchTS = { #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N .xsymm_iutcopy = xsymm_iutcopyTS, .xsymm_iltcopy = xsymm_iltcopyTS, @@ -1588,6 +2553,11 @@ gotoblas_t TABLE_NAME = { #endif .xsymm_outcopy = xsymm_outcopyTS, .xsymm_oltcopy = xsymm_oltcopyTS, +}; +#endif + +#ifdef EXPRECISION +const openblas_xhemm_dispatch_t openblas_xhemm_dispatchTS = { #if XGEMM_DEFAULT_UNROLL_M != XGEMM_DEFAULT_UNROLL_N .xhemm_iutcopy = xhemm_iutcopyTS, .xhemm_iltcopy = xhemm_iltcopyTS, @@ -1597,17 +2567,13 @@ gotoblas_t TABLE_NAME = { #endif .xhemm_outcopy = xhemm_outcopyTS, .xhemm_oltcopy = xhemm_oltcopyTS, +}; +#endif - .xgemm3m_p = 0, - .xgemm3m_q = 0, - .xgemm3m_r = 0, +#ifdef EXPRECISION +const openblas_xgemm3m_dispatch_t openblas_xgemm3m_dispatchTS = { #if (USE_GEMM3M) - .xgemm3m_unroll_m = QGEMM_DEFAULT_UNROLL_M, - .xgemm3m_unroll_n = QGEMM_DEFAULT_UNROLL_N, - .xgemm3m_unroll_mn = MAX(QGEMM_DEFAULT_UNROLL_M, QGEMM_DEFAULT_UNROLL_N), - .xgemm3m_kernel = xgemm3m_kernelTS, - .xgemm3m_incopyb = xgemm3m_incopybTS, .xgemm3m_incopyr = xgemm3m_incopyrTS, .xgemm3m_incopyi = xgemm3m_incopyiTS, @@ -1620,40 +2586,8 @@ gotoblas_t TABLE_NAME = { .xgemm3m_otcopyb = xgemm3m_otcopybTS, .xgemm3m_otcopyr = xgemm3m_otcopyrTS, .xgemm3m_otcopyi = xgemm3m_otcopyiTS, - - .xsymm3m_iucopyb = xsymm3m_iucopybTS, - .xsymm3m_ilcopyb = xsymm3m_ilcopybTS, - .xsymm3m_iucopyr = xsymm3m_iucopyrTS, - .xsymm3m_ilcopyr = xsymm3m_ilcopyrTS, - .xsymm3m_iucopyi = xsymm3m_iucopyiTS, - .xsymm3m_ilcopyi = xsymm3m_ilcopyiTS, - .xsymm3m_oucopyb = xsymm3m_oucopybTS, - .xsymm3m_olcopyb = xsymm3m_olcopybTS, - .xsymm3m_oucopyr = xsymm3m_oucopyrTS, - .xsymm3m_olcopyr = xsymm3m_olcopyrTS, - .xsymm3m_oucopyi = xsymm3m_oucopyiTS, - .xsymm3m_olcopyi = xsymm3m_olcopyiTS, - - .xhemm3m_iucopyb = xhemm3m_iucopybTS, - .xhemm3m_ilcopyb = xhemm3m_ilcopybTS, - .xhemm3m_iucopyr = xhemm3m_iucopyrTS, - .xhemm3m_ilcopyr = xhemm3m_ilcopyrTS, - .xhemm3m_iucopyi = xhemm3m_iucopyiTS, - .xhemm3m_ilcopyi = xhemm3m_ilcopyiTS, - - .xhemm3m_oucopyb = xhemm3m_oucopybTS, - .xhemm3m_olcopyb = xhemm3m_olcopybTS, - .xhemm3m_oucopyr = xhemm3m_oucopyrTS, - .xhemm3m_olcopyr = xhemm3m_olcopyrTS, - .xhemm3m_oucopyi = xhemm3m_oucopyiTS, - .xhemm3m_olcopyi = xhemm3m_olcopyiTS, #else - .xgemm3m_unroll_m = 0, - .xgemm3m_unroll_n = 0, - .xgemm3m_unroll_mn = 0, - .xgemm3m_kernel = NULL, - .xgemm3m_incopyb = NULL, .xgemm3m_incopyr = NULL, .xgemm3m_incopyi = NULL, @@ -1666,7 +2600,26 @@ gotoblas_t TABLE_NAME = { .xgemm3m_otcopyb = NULL, .xgemm3m_otcopyr = NULL, .xgemm3m_otcopyi = NULL, +#endif +}; +#endif +#ifdef EXPRECISION +const openblas_xsymm3m_dispatch_t openblas_xsymm3m_dispatchTS = { +#if (USE_GEMM3M) + .xsymm3m_iucopyb = xsymm3m_iucopybTS, + .xsymm3m_ilcopyb = xsymm3m_ilcopybTS, + .xsymm3m_iucopyr = xsymm3m_iucopyrTS, + .xsymm3m_ilcopyr = xsymm3m_ilcopyrTS, + .xsymm3m_iucopyi = xsymm3m_iucopyiTS, + .xsymm3m_ilcopyi = xsymm3m_ilcopyiTS, + .xsymm3m_oucopyb = xsymm3m_oucopybTS, + .xsymm3m_olcopyb = xsymm3m_olcopybTS, + .xsymm3m_oucopyr = xsymm3m_oucopyrTS, + .xsymm3m_olcopyr = xsymm3m_olcopyrTS, + .xsymm3m_oucopyi = xsymm3m_oucopyiTS, + .xsymm3m_olcopyi = xsymm3m_olcopyiTS, +#else .xsymm3m_iucopyb = NULL, .xsymm3m_ilcopyb = NULL, .xsymm3m_iucopyr = NULL, @@ -1679,14 +2632,32 @@ gotoblas_t TABLE_NAME = { .xsymm3m_olcopyr = NULL, .xsymm3m_oucopyi = NULL, .xsymm3m_olcopyi = NULL, +#endif +}; +#endif +#ifdef EXPRECISION +const openblas_xhemm3m_dispatch_t openblas_xhemm3m_dispatchTS = { +#if (USE_GEMM3M) + .xhemm3m_iucopyb = xhemm3m_iucopybTS, + .xhemm3m_ilcopyb = xhemm3m_ilcopybTS, + .xhemm3m_iucopyr = xhemm3m_iucopyrTS, + .xhemm3m_ilcopyr = xhemm3m_ilcopyrTS, + .xhemm3m_iucopyi = xhemm3m_iucopyiTS, + .xhemm3m_ilcopyi = xhemm3m_ilcopyiTS, + .xhemm3m_oucopyb = xhemm3m_oucopybTS, + .xhemm3m_olcopyb = xhemm3m_olcopybTS, + .xhemm3m_oucopyr = xhemm3m_oucopyrTS, + .xhemm3m_olcopyr = xhemm3m_olcopyrTS, + .xhemm3m_oucopyi = xhemm3m_oucopyiTS, + .xhemm3m_olcopyi = xhemm3m_olcopyiTS, +#else .xhemm3m_iucopyb = NULL, .xhemm3m_ilcopyb = NULL, .xhemm3m_iucopyr = NULL, .xhemm3m_ilcopyr = NULL, .xhemm3m_iucopyi = NULL, .xhemm3m_ilcopyi = NULL, - .xhemm3m_oucopyb = NULL, .xhemm3m_olcopyb = NULL, .xhemm3m_oucopyr = NULL, @@ -1694,48 +2665,73 @@ gotoblas_t TABLE_NAME = { .xhemm3m_oucopyi = NULL, .xhemm3m_olcopyi = NULL, #endif +}; +#endif +#ifdef EXPRECISION +const openblas_xneg_dispatch_t openblas_xneg_dispatchTS = { #ifndef NO_LAPACK .xneg_tcopy = xneg_tcopyTS, - .xlaswp_ncopy = xlaswp_ncopyTS, #else .xneg_tcopy = NULL, - .xlaswp_ncopy = NULL, #endif - +}; #endif - .init = init_parameter, +#ifdef EXPRECISION +const openblas_xlaswp_dispatch_t openblas_xlaswp_dispatchTS = { +#ifndef NO_LAPACK + .xlaswp_ncopy = xlaswp_ncopyTS, +#else + .xlaswp_ncopy = NULL, +#endif +}; +#endif - .snum_opt = SNUMOPT, - .dnum_opt = DNUMOPT, - .qnum_opt = QNUMOPT, -#if BUILD_SINGLE == 1 +#if (BUILD_SINGLE==1) +const openblas_saxpby_dispatch_t openblas_saxpby_dispatchTS = { .saxpby_k = saxpby_kTS, +}; #endif -#if BUILD_DOUBLE == 1 + +#if (BUILD_DOUBLE==1) +const openblas_daxpby_dispatch_t openblas_daxpby_dispatchTS = { .daxpby_k = daxpby_kTS, +}; #endif -#if BUILD_COMPLEX == 1 + +#if (BUILD_COMPLEX==1) +const openblas_caxpby_dispatch_t openblas_caxpby_dispatchTS = { .caxpby_k = caxpby_kTS, +}; #endif -#if BUILD_COMPLEX16== 1 + +#if (BUILD_COMPLEX16==1) +const openblas_zaxpby_dispatch_t openblas_zaxpby_dispatchTS = { .zaxpby_k = zaxpby_kTS, +}; #endif -#if BUILD_SINGLE == 1 +#if (BUILD_SINGLE==1) +const openblas_somatcopy_dispatch_t openblas_somatcopy_dispatchTS = { .somatcopy_k_cn = somatcopy_k_cnTS, .somatcopy_k_ct = somatcopy_k_ctTS, .somatcopy_k_rn = somatcopy_k_rnTS, .somatcopy_k_rt = somatcopy_k_rtTS, +}; #endif -#if BUILD_DOUBLE== 1 + +#if (BUILD_DOUBLE==1) +const openblas_domatcopy_dispatch_t openblas_domatcopy_dispatchTS = { .domatcopy_k_cn = domatcopy_k_cnTS, .domatcopy_k_ct = domatcopy_k_ctTS, .domatcopy_k_rn = domatcopy_k_rnTS, .domatcopy_k_rt = domatcopy_k_rtTS, +}; #endif -#if BUILD_COMPLEX == 1 + +#if (BUILD_COMPLEX==1) +const openblas_comatcopy_dispatch_t openblas_comatcopy_dispatchTS = { .comatcopy_k_cn = comatcopy_k_cnTS, .comatcopy_k_ct = comatcopy_k_ctTS, .comatcopy_k_rn = comatcopy_k_rnTS, @@ -1744,8 +2740,11 @@ gotoblas_t TABLE_NAME = { .comatcopy_k_ctc = comatcopy_k_ctcTS, .comatcopy_k_rnc = comatcopy_k_rncTS, .comatcopy_k_rtc = comatcopy_k_rtcTS, +}; #endif -#if BUILD_COMPLEX16 == 1 + +#if (BUILD_COMPLEX16==1) +const openblas_zomatcopy_dispatch_t openblas_zomatcopy_dispatchTS = { .zomatcopy_k_cn = zomatcopy_k_cnTS, .zomatcopy_k_ct = zomatcopy_k_ctTS, .zomatcopy_k_rn = zomatcopy_k_rnTS, @@ -1754,21 +2753,29 @@ gotoblas_t TABLE_NAME = { .zomatcopy_k_ctc = zomatcopy_k_ctcTS, .zomatcopy_k_rnc = zomatcopy_k_rncTS, .zomatcopy_k_rtc = zomatcopy_k_rtcTS, +}; #endif -#if BUILD_SINGLE == 1 +#if (BUILD_SINGLE==1) +const openblas_simatcopy_dispatch_t openblas_simatcopy_dispatchTS = { .simatcopy_k_cn = simatcopy_k_cnTS, .simatcopy_k_ct = simatcopy_k_ctTS, .simatcopy_k_rn = simatcopy_k_rnTS, .simatcopy_k_rt = simatcopy_k_rtTS, +}; #endif -#if BUILD_DOUBLE== 1 + +#if (BUILD_DOUBLE==1) +const openblas_dimatcopy_dispatch_t openblas_dimatcopy_dispatchTS = { .dimatcopy_k_cn = dimatcopy_k_cnTS, .dimatcopy_k_ct = dimatcopy_k_ctTS, .dimatcopy_k_rn = dimatcopy_k_rnTS, .dimatcopy_k_rt = dimatcopy_k_rtTS, +}; #endif -#if BUILD_COMPLEX== 1 + +#if (BUILD_COMPLEX==1) +const openblas_cimatcopy_dispatch_t openblas_cimatcopy_dispatchTS = { .cimatcopy_k_cn = cimatcopy_k_cnTS, .cimatcopy_k_ct = cimatcopy_k_ctTS, .cimatcopy_k_rn = cimatcopy_k_rnTS, @@ -1777,8 +2784,11 @@ gotoblas_t TABLE_NAME = { .cimatcopy_k_ctc = cimatcopy_k_ctcTS, .cimatcopy_k_rnc = cimatcopy_k_rncTS, .cimatcopy_k_rtc = cimatcopy_k_rtcTS, +}; #endif -#if BUILD_COMPLEX16==1 + +#if (BUILD_COMPLEX16==1) +const openblas_zimatcopy_dispatch_t openblas_zimatcopy_dispatchTS = { .zimatcopy_k_cn = zimatcopy_k_cnTS, .zimatcopy_k_ct = zimatcopy_k_ctTS, .zimatcopy_k_rn = zimatcopy_k_rnTS, @@ -1787,21 +2797,32 @@ gotoblas_t TABLE_NAME = { .zimatcopy_k_ctc = zimatcopy_k_ctcTS, .zimatcopy_k_rnc = zimatcopy_k_rncTS, .zimatcopy_k_rtc = zimatcopy_k_rtcTS, +}; #endif -#if BUILD_SINGLE == 1 +#if (BUILD_SINGLE==1) +const openblas_sgeadd_dispatch_t openblas_sgeadd_dispatchTS = { .sgeadd_k = sgeadd_kTS, +}; #endif -#if BUILD_DOUBLE==1 + +#if (BUILD_DOUBLE==1) +const openblas_dgeadd_dispatch_t openblas_dgeadd_dispatchTS = { .dgeadd_k = dgeadd_kTS, +}; #endif -#if BUILD_COMPLEX==1 + +#if (BUILD_COMPLEX==1) +const openblas_cgeadd_dispatch_t openblas_cgeadd_dispatchTS = { .cgeadd_k = cgeadd_kTS, +}; #endif -#if BUILD_COMPLEX16==1 + +#if (BUILD_COMPLEX16==1) +const openblas_zgeadd_dispatch_t openblas_zgeadd_dispatchTS = { .zgeadd_k = zgeadd_kTS, -#endif }; +#endif #if (ARCH_ARM64) static void init_parameter(void) { From 37da1767be307498f32819fbcd9c5d3a0f3877a7 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 1 Oct 2026 19:36:16 +0000 Subject: [PATCH 7/7] Remove FUNC_OFFSET and FUNC_BASE Nothing uses them since the kernels moved out of gotoblas_t: the small-matrix kernel tables use OPENBLAS_DISPATCH_OFFSET/_BASE instead. Co-Authored-By: Claude Opus 5.5 --- common_param.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common_param.h b/common_param.h index 71d8a0dc52..d5fb7ffc20 100644 --- a/common_param.h +++ b/common_param.h @@ -2489,10 +2489,6 @@ typedef struct { extern const openblas_zgeadd_dispatch_t *const openblas_zgeadd_dispatch[OPENBLAS_NUM_CORES]; #endif -#define FUNC_OFFSET(func) (size_t)(&((gotoblas_t *)NULL)->func) -/* The address that FUNC_OFFSET(func) is an offset from. */ -#define FUNC_BASE(func) ((char *)gotoblas) - #define DTB_ENTRIES gotoblas -> dtb_entries #define GEMM_OFFSET_A gotoblas -> offsetA #define GEMM_OFFSET_B gotoblas -> offsetB @@ -2630,7 +2626,6 @@ extern const openblas_zgeadd_dispatch_t *const openblas_zgeadd_dispatch[OPENBLAS #else -#define FUNC_OFFSET(func) (size_t)(func) #define OPENBLAS_DISPATCH_OFFSET(group, func) (size_t)(func) #define DTB_ENTRIES DTB_DEFAULT_ENTRIES