diff --git a/level_zero/core/source/helpers/api_specific_config_l0.cpp b/level_zero/core/source/helpers/api_specific_config_l0.cpp index e28b2238ef749..01e281000f407 100644 --- a/level_zero/core/source/helpers/api_specific_config_l0.cpp +++ b/level_zero/core/source/helpers/api_specific_config_l0.cpp @@ -115,5 +115,9 @@ bool ApiSpecificConfig::isGlobalStatelessEnabled(const RootDeviceEnvironment &ro bool ApiSpecificConfig::isUpdateTagFromWaitEnabledForHeapless() { return false; } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &device) { + return getGlobalBindlessHeapConfiguration(device.getHardwareInfo()) && getBindlessMode(device); +} + } // namespace NEO diff --git a/opencl/extensions/public/cl_ext_private.h b/opencl/extensions/public/cl_ext_private.h index 7ee6d576b40a3..0a3f839315f4b 100644 --- a/opencl/extensions/public/cl_ext_private.h +++ b/opencl/extensions/public/cl_ext_private.h @@ -421,4 +421,11 @@ typedef struct _cl_kernel_allocation_info_intel { // cl_device bfloat16 atomic capabilities #if !defined(CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT) #define CL_DEVICE_BFLOAT16_FP_ATOMIC_CAPABILITIES_EXT 0x10012 + +/****************************************************** + * cl_intel_bindless_images extension * + ******************************************************/ +#define CL_MEM_BINDLESS_IMAGE_INTEL 0x10060 +#define CL_IMAGE_BINDLESS_HANDLE_INTEL 0x10061 + #endif diff --git a/opencl/source/command_queue/enqueue_common.h b/opencl/source/command_queue/enqueue_common.h index 67116fc75167e..43c268a3fce4d 100644 --- a/opencl/source/command_queue/enqueue_common.h +++ b/opencl/source/command_queue/enqueue_common.h @@ -937,6 +937,7 @@ CompletionStamp CommandQueueHw::enqueueNonBlocked( isDcFlushRequiredOnStallingCommandsOnNextFlush() // isDcFlushRequiredOnStallingCommandsOnNextFlush ); + dispatchFlags.useBindlessImages = kernel->usesBindlessImages(); dispatchFlags.isWalkerWithProfilingEnqueued = getAndClearIsWalkerWithProfilingEnqueued(); dispatchFlags.pipelineSelectArgs.systolicPipelineSelectMode = systolicPipelineSelectMode; uint32_t lws[3] = {static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().x), static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().y), static_cast(multiDispatchInfo.begin()->getLocalWorkgroupSize().z)}; diff --git a/opencl/source/helpers/api_specific_config_ocl.cpp b/opencl/source/helpers/api_specific_config_ocl.cpp index 5cdcf601ee9b8..82f7dfa9023d5 100644 --- a/opencl/source/helpers/api_specific_config_ocl.cpp +++ b/opencl/source/helpers/api_specific_config_ocl.cpp @@ -9,6 +9,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/compiler_product_helper.h" +#include "shared/source/helpers/hw_info.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -19,7 +20,7 @@ namespace NEO { StackVec validClPrefixes; StackVec validClPrefixTypes; -bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const HardwareInfo &hwInfo) { +bool ApiSpecificConfig::getGlobalBindlessHeapConfiguration(const HardwareInfo &) { return false; } @@ -27,12 +28,10 @@ bool ApiSpecificConfig::getBindlessMode(const Device &device) { if (device.getCompilerProductHelper().isHeaplessModeEnabled(device.getHardwareInfo())) { return true; } - if (debugManager.flags.UseBindlessMode.get() != -1) { return debugManager.flags.UseBindlessMode.get(); - } else { - return false; } + return false; } bool ApiSpecificConfig::isDeviceAllocationCacheEnabled() { @@ -96,4 +95,8 @@ bool ApiSpecificConfig::isUpdateTagFromWaitEnabledForHeapless() { return true; } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &) { + return false; +} + } // namespace NEO diff --git a/opencl/source/helpers/cl_memory_properties_helpers.cpp b/opencl/source/helpers/cl_memory_properties_helpers.cpp index 68843602c6564..61f97e36d1924 100644 --- a/opencl/source/helpers/cl_memory_properties_helpers.cpp +++ b/opencl/source/helpers/cl_memory_properties_helpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2021-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int uint64_t handleType = 0; uintptr_t hostptr = 0; std::vector devices; - + bool bindlessImage = false; if (properties != nullptr) { for (int i = 0; properties[i] != 0; i += 2) { switch (properties[i]) { @@ -73,6 +73,9 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int i++; } break; + case CL_MEM_BINDLESS_IMAGE_INTEL: // Avoiding get a false when using bindless image extension + bindlessImage = true; + break; default: return false; } @@ -84,7 +87,7 @@ bool ClMemoryPropertiesHelper::parseMemoryProperties(const cl_mem_properties_int memoryProperties.handle = handle; memoryProperties.hostptr = hostptr; memoryProperties.associatedDevices = devices; - + memoryProperties.flags.bindlessImage = bindlessImage; switch (objectType) { case ClMemoryPropertiesHelper::ObjType::buffer: return isFieldValid(flags, MemObjHelper::validFlagsForBuffer) && diff --git a/opencl/source/helpers/hardware_commands_helper_base.inl b/opencl/source/helpers/hardware_commands_helper_base.inl index 3758eb4f33224..22032363278d5 100644 --- a/opencl/source/helpers/hardware_commands_helper_base.inl +++ b/opencl/source/helpers/hardware_commands_helper_base.inl @@ -13,6 +13,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/address_patch.h" #include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" @@ -260,9 +261,10 @@ size_t HardwareCommandsHelper::sendIndirectState( const auto &samplerTable = kernelInfo.kernelDescriptor.payloadMappings.samplerTable; if (isValidOffset(samplerTable.tableOffset) && isValidOffset(samplerTable.borderColor)) { samplerCount = samplerTable.numSamplers; + auto *samplerHeapHelper = ApiSpecificConfig::getBindlessMode(device) ? device.getBindlessHeapsHelper() : nullptr; samplerStateOffset = EncodeStates::copySamplerState(&dsh, samplerTable.tableOffset, samplerCount, samplerTable.borderColor, - kernel.getDynamicStateHeap(), device.getBindlessHeapsHelper(), + kernel.getDynamicStateHeap(), samplerHeapHelper, device.getRootDeviceEnvironment()); if constexpr (heaplessModeEnabled) { uint64_t bindlessSamplerStateAddress = samplerStateOffset; diff --git a/opencl/source/helpers/task_information.cpp b/opencl/source/helpers/task_information.cpp index d37ac6da0ba4d..a9e3a084cb926 100644 --- a/opencl/source/helpers/task_information.cpp +++ b/opencl/source/helpers/task_information.cpp @@ -216,6 +216,7 @@ CompletionStamp &CommandComputeKernel::submit(TaskCountType taskLevel, bool term commandQueue.isStallingCommandsOnNextFlushRequired(), // isStallingCommandsOnNextFlushRequired commandQueue.isDcFlushRequiredOnStallingCommandsOnNextFlush() // isDcFlushRequiredOnStallingCommandsOnNextFlush ); + dispatchFlags.useBindlessImages = kernel->usesBindlessImages(); if (commandQueue.getContext().getRootDeviceIndices().size() > 1) { eventsRequest.fillCsrDependenciesForRootDevices(dispatchFlags.csrDependencies, commandStreamReceiver); } diff --git a/opencl/source/kernel/kernel.cpp b/opencl/source/kernel/kernel.cpp index effd6e224278d..fef6352732110 100644 --- a/opencl/source/kernel/kernel.cpp +++ b/opencl/source/kernel/kernel.cpp @@ -18,6 +18,7 @@ #include "shared/source/gmm_helper/resource_info.h" #include "shared/source/helpers/address_patch.h" #include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/helpers/compiler_product_helper.h" @@ -30,6 +31,7 @@ #include "shared/source/helpers/ptr_math.h" #include "shared/source/helpers/simd_helper.h" #include "shared/source/helpers/surface_format_info.h" +#include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/kernel/local_ids_cache.h" #include "shared/source/memory_manager/compression_selector.h" #include "shared/source/memory_manager/memory_manager.h" @@ -147,9 +149,9 @@ void Kernel::patchWithImplicitSurface(uint64_t ptrToPatchInCrossThreadData, Grap void *surfaceState = nullptr; auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(clDevice.getRootDeviceEnvironment()); - if (clDevice.getDevice().getBindlessHeapsHelper()) { + if (clDevice.getDevice().getBindlessHeapsHelper() && + ApiSpecificConfig::getBindlessMode(clDevice.getDevice())) { UNRECOVERABLE_IF(clDevice.getDevice().getCompilerProductHelper().isHeaplessModeEnabled(clDevice.getHardwareInfo())); - auto &ssInHeap = allocation.getBindlessInfo(); surfaceState = ssInHeap.ssPtr; auto patchLocation = ptrOffset(crossThreadData, arg.bindless); @@ -1341,7 +1343,17 @@ void Kernel::makeResident(CommandStreamReceiver &commandStreamReceiver) { pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(this->getContext().getSVMAllocsManager()); } makeArgsResident(commandStreamReceiver); - + auto bindlessHelper = getDevice().getDevice().getBindlessHeapsHelper(); + if (bindlessHelper) { + for (auto heapType : {NEO::BindlessHeapsHelper::specialSsh, + NEO::BindlessHeapsHelper::globalSsh, + NEO::BindlessHeapsHelper::globalDsh}) { + auto heap = bindlessHelper->getHeap(heapType); + if (heap) { + commandStreamReceiver.makeResident(*heap->getGraphicsAllocation()); + } + } + } auto kernelIsaAllocation = this->kernelInfo.getIsaGraphicsAllocation(); if (kernelIsaAllocation) { commandStreamReceiver.makeResident(*kernelIsaAllocation); @@ -1523,6 +1535,10 @@ void Kernel::getAllocationsInfo(std::vector &al } } +bool Kernel::usesBindlessImages() const { + return program->usesBindlessImages(clDevice.getRootDeviceIndex()); +} + cl_int Kernel::setArgLocal(uint32_t argIndexIn, size_t argSize, const void *argVal) { @@ -1715,7 +1731,19 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, imageFromBufferArgsCount += (pImage->isImageFromBuffer() ? 1 : 0) - (wasImageFromBuffer ? 1 : 0); void *surfaceState = nullptr; - if (isValidOffset(argAsImg.bindless)) { + if (isValidOffset(argAsImg.bindless) && pImage->isBindlessImage()) { + auto bindlessSlot = pImage->getBindlessSlot(); + if (bindlessSlot && bindlessSlot->ssPtr) { + surfaceState = bindlessSlot->ssPtr; + + auto &gfxCoreHelper = this->getGfxCoreHelper(); + auto patchLocation = ptrOffset(getCrossThreadData(), argAsImg.bindless); + uint64_t patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue( + static_cast(bindlessSlot->surfaceStateOffset)); + uint32_t patchSize = NEO::isUndefinedOffset(argAsImg.size) ? 0 : argAsImg.size; + patchWithRequiredSize(reinterpret_cast(patchLocation), patchSize, patchValue); + } + } else if (isValidOffset(argAsImg.bindless)) { auto ssIndex = getSurfaceStateIndexForBindlessOffset(argAsImg.bindless); if (ssIndex < std::numeric_limits::max()) { auto &gfxCoreHelper = this->getGfxCoreHelper(); @@ -1727,10 +1755,8 @@ cl_int Kernel::setArgImageWithMipLevel(uint32_t argIndex, surfaceState = ptrOffset(getSurfaceStateHeap(), argAsImg.bindful); } - // Sets SS structure UNRECOVERABLE_IF(surfaceState == nullptr); pImage->setImageArg(surfaceState, arg.getExtendedTypeInfo().isMediaBlockImage, mipLevel, rootDeviceIndex); - auto &imageDesc = pImage->getImageDesc(); auto &imageFormat = pImage->getImageFormat(); @@ -2075,13 +2101,22 @@ void Kernel::patchBindlessSurfaceStatesInCrossThreadData(uint64_t bindlessSurfac auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(getDevice().getRootDeviceEnvironment()); auto *crossThreadDataPtr = reinterpret_cast(getCrossThreadData()); - for (auto &arg : kernelInfo.kernelDescriptor.payloadMappings.explicitArgs) { + const auto &explicitArgs = kernelInfo.kernelDescriptor.payloadMappings.explicitArgs; + for (size_t argIndex = 0; argIndex < explicitArgs.size(); argIndex++) { + const auto &arg = explicitArgs[argIndex]; auto offset = NEO::undefined; if (arg.type == NEO::ArgDescriptor::argTPointer) { offset = arg.as().bindless; } else if (arg.type == NEO::ArgDescriptor::argTImage) { offset = arg.as().bindless; + if (NEO::isValidOffset(offset)) { + auto clMem = static_cast(kernelArguments[argIndex].object); + auto pImage = castToObject(clMem); + if (pImage && pImage->isBindlessImage()) { + continue; + } + } } else { continue; } diff --git a/opencl/source/kernel/kernel.h b/opencl/source/kernel/kernel.h index ebc2a6c095a6c..5a07b170abc8f 100644 --- a/opencl/source/kernel/kernel.h +++ b/opencl/source/kernel/kernel.h @@ -391,6 +391,8 @@ class Kernel : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova MOCKABLE_VIRTUAL void getAllocationsInfo(std::vector &allocationsInfo) const; + bool usesBindlessImages() const; + protected: Kernel(Program *programArg, const KernelInfo &kernelInfo, ClDevice &clDevice); diff --git a/opencl/source/mem_obj/image.cpp b/opencl/source/mem_obj/image.cpp index 0f6607954ebf2..034af36bc54cb 100644 --- a/opencl/source/mem_obj/image.cpp +++ b/opencl/source/mem_obj/image.cpp @@ -301,6 +301,30 @@ Image *Image::create(Context *context, setImageProperties(image, *imageDesc, imgInfo, parentImage, parentBuffer, hostPtrRowPitch, hostPtrSlicePitch, imageCount, hostPtrMinSize); + auto defaultRootDeviceEnv = defaultDevice->getExecutionEnvironment()->rootDeviceEnvironments[defaultRootDeviceIndex].get(); + + if (image && memoryProperties.flags.bindlessImage && + defaultRootDeviceEnv->getBindlessHeapsHelper() == nullptr) { + auto *rootDevice = defaultDevice->getRootDevice(); + defaultRootDeviceEnv->createBindlessHeapsHelper( + rootDevice, rootDevice->getNumGenericSubDevices() > 1); + } + + auto bindlessHelper = defaultRootDeviceEnv->getBindlessHeapsHelper(); + if (bindlessHelper && image && memoryProperties.flags.bindlessImage) { + auto allocation = image->getGraphicsAllocation(defaultRootDeviceIndex); + auto memManager = context->getMemoryManager(); + if (memManager->allocateBindlessSlot(allocation)) { + if (allocation->getBindlessOffset() != std::numeric_limits::max()) { + image->bindlessInfo = std::make_unique(allocation->getBindlessInfo()); + image->bindlessImage = true; + if (image->bindlessInfo->ssPtr) { + image->setImageArg(image->bindlessInfo->ssPtr, false, 0, defaultRootDeviceIndex); + } + } + } + } + errcodeRet = CL_SUCCESS; auto &defaultHwInfo = defaultDevice->getHardwareInfo(); if (context->isProvidingPerformanceHints()) { @@ -870,7 +894,11 @@ cl_int Image::getImageInfo(cl_image_info paramName, srcParamSize = sizeof(cl_uint); srcParam = &(imageDesc.num_samples); break; - + case CL_IMAGE_BINDLESS_HANDLE_INTEL: + srcParamSize = sizeof(uint64_t); + retParam = getBindlessHandle(); + srcParam = &retParam; + break; default: getOsSpecificImageInfo(paramName, &srcParamSize, &srcParam); break; diff --git a/opencl/source/mem_obj/image.h b/opencl/source/mem_obj/image.h index eaad5a8de5240..3627a8d68a380 100644 --- a/opencl/source/mem_obj/image.h +++ b/opencl/source/mem_obj/image.h @@ -6,6 +6,7 @@ */ #pragma once +#include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/memory_manager/graphics_allocation.h" #include "opencl/source/helpers/surface_formats.h" @@ -209,6 +210,9 @@ class Image : public MemObj { void fillImageRegion(size_t *region) const; static bool validateHandleType(MemoryProperties &memoryProperties, UnifiedSharingMemoryDescription &extMem); + SurfaceStateInHeapInfo *getBindlessSlot() const { return bindlessInfo.get(); } + uint64_t getBindlessHandle() const { return bindlessInfo ? bindlessInfo->surfaceStateOffset : 0; } + bool isBindlessImage() const { return bindlessImage; } void setAs3DUavOrRtvImage(bool isUavOrRtv); void setIsPackedFormat(bool isPackedFormat) { this->isPackedFormat = isPackedFormat; } @@ -252,6 +256,8 @@ class Image : public MemObj { ImagePlane plane = ImagePlane::noPlane; bool is3DUAVOrRTV = false; bool isPackedFormat = false; + std::unique_ptr bindlessInfo; + bool bindlessImage = false; static bool isValidSingleChannelFormat(const cl_image_format *imageFormat); static bool isValidIntensityFormat(const cl_image_format *imageFormat); diff --git a/opencl/source/program/build.cpp b/opencl/source/program/build.cpp index 99e7ed863d0db..9165c381e94bf 100644 --- a/opencl/source/program/build.cpp +++ b/opencl/source/program/build.cpp @@ -55,7 +55,15 @@ cl_int Program::build( options = ""; } } + const bool hasBindlessImages = CompilerOptions::extract(CompilerOptions::bindlessImages, options); + const bool hasBindlessAdvancedMode = CompilerOptions::extract(CompilerOptions::bindlessAdvancedMode, options); + if (hasBindlessImages) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessImages); + } + if (hasBindlessAdvancedMode) { + CompilerOptions::concatenateAppend(internalOptions, CompilerOptions::bindlessAdvancedMode); + } const bool shouldSuppressRebuildWarning{CompilerOptions::extract(CompilerOptions::noRecompiledFromIr, options)}; extractInternalOptions(options, internalOptions); CompilerOptions::applyAdditionalApiOptions(options); @@ -133,6 +141,9 @@ cl_int Program::build( } this->replaceDeviceBinary(std::move(compilerOuput.deviceBinary.mem), compilerOuput.deviceBinary.size, clDevice->getRootDeviceIndex()); phaseReached[clDevice->getRootDeviceIndex()] = BuildPhase::binaryCreation; + + this->buildInfos[clDevice->getRootDeviceIndex()].useBindlessImages = hasBindlessImages; + } if (retVal != CL_SUCCESS) { break; diff --git a/opencl/source/program/program.h b/opencl/source/program/program.h index 7b25d64920e91..b4ab89d0219e8 100644 --- a/opencl/source/program/program.h +++ b/opencl/source/program/program.h @@ -193,6 +193,10 @@ class Program : public BaseObject<_cl_program> { return isSpirV; } + bool usesBindlessImages(uint32_t rootDeviceIndex) const { + return buildInfos[rootDeviceIndex].useBindlessImages; + } + void freeGlobalBufferAllocation(std::unique_ptr &buffer); NEO::SharedPoolAllocation *getConstantSurface(uint32_t rootDeviceIndex) const; @@ -381,6 +385,8 @@ class Program : public BaseObject<_cl_program> { std::unique_ptr debugData; size_t debugDataSize = 0U; size_t kernelMiscInfoPos = std::string::npos; + + bool useBindlessImages = false; }; std::vector buildInfos; diff --git a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp index 10547f82f8d33..93cdc9781cf47 100644 --- a/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp +++ b/opencl/test/unit_test/helpers/api_specific_config_ocl_tests.cpp @@ -6,9 +6,11 @@ */ #include "shared/source/helpers/api_specific_config.h" +#include "shared/source/helpers/hw_info.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/compression_selector.h" -#include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/mocks/mock_device.h" +#include "shared/test/common/mocks/ult_device_factory.h" #include "opencl/source/os_interface/ocl_reg_path.h" @@ -57,4 +59,19 @@ TEST(ApiSpecificConfigOclTests, WhenCheckingIfDeviceUsmPoolingIsEnabledThenRetur EXPECT_TRUE(ApiSpecificConfig::isDeviceUsmPoolingEnabled()); } +TEST(ApiSpecificConfigOclTests, WhenGettingGlobalBindlessHeapConfigurationThenReturnFalse) { + UltDeviceFactory deviceFactory{1, 0}; + auto *device = deviceFactory.rootDevices[0]; + auto &hwInfo = device->getHardwareInfo(); + + EXPECT_FALSE(ApiSpecificConfig::getGlobalBindlessHeapConfiguration(hwInfo)); +} + +TEST(ApiSpecificConfigOclTests, WhenCreatingBindlessHeapsHelperThenReturnFalse) { + UltDeviceFactory deviceFactory{1, 0}; + auto *device = deviceFactory.rootDevices[0]; + + auto result = ApiSpecificConfig::createBindlessHeapsHelper(*device); + EXPECT_FALSE(result); +} } // namespace NEO diff --git a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp index ca4b1d3c2ee03..7c950592ae2db 100644 --- a/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp +++ b/opencl/test/unit_test/helpers/cl_memory_properties_helpers_tests.cpp @@ -14,6 +14,7 @@ #include "shared/test/common/mocks/mock_graphics_allocation.h" #include "shared/test/common/mocks/ult_device_factory.h" +#include "opencl/extensions/public/cl_ext_private.h" #include "opencl/source/helpers/cl_memory_properties_helpers.h" #include "opencl/source/mem_obj/mem_obj_helper.h" #include "opencl/test/unit_test/mocks/mock_cl_device.h" @@ -599,3 +600,20 @@ TEST_F(MemoryPropertiesHelperTests, givenSubDeviceIdWhenParsingExtraMemoryProper EXPECT_EQ(0b10u, memoryProperties.pDevice->getDeviceBitfield().to_ulong()); EXPECT_EQ(&context.pSubDevice1->getDevice(), memoryProperties.pDevice); } +TEST_F(MemoryPropertiesHelperTests, givenBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenTrueIsReturnedAndFlagIsSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_BINDLESS_IMAGE_INTEL, 1, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_TRUE(memoryProperties.flags.bindlessImage); +} + +TEST_F(MemoryPropertiesHelperTests, givenNoBindlessImagePropertyWhenParsingMemoryPropertiesForImageThenBindlessFlagIsNotSet) { + cl_mem_properties_intel properties[] = { + CL_MEM_FLAGS, CL_MEM_READ_WRITE, + 0}; + EXPECT_TRUE(ClMemoryPropertiesHelper::parseMemoryProperties(properties, memoryProperties, flags, flagsIntel, allocflags, + ClMemoryPropertiesHelper::ObjType::image, context)); + EXPECT_FALSE(memoryProperties.flags.bindlessImage); +} diff --git a/opencl/test/unit_test/kernel/kernel_tests.cpp b/opencl/test/unit_test/kernel/kernel_tests.cpp index b9f5167af9848..da9cb0c8bf819 100644 --- a/opencl/test/unit_test/kernel/kernel_tests.cpp +++ b/opencl/test/unit_test/kernel/kernel_tests.cpp @@ -485,6 +485,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchingCrossThreadDataThenCo pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.implicitArgs.globalConstantsSurfaceAddress.bindless = 4 * sizeof(uint64_t); MockKernel mockKernel(pProgram, pProgram->mockKernelInfo, *pClDevice); + mockKernel.kernelArguments.resize(pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.explicitArgs.size()); pProgram->mockKernelInfo.kernelDescriptor.initBindlessOffsetToSurfaceState(); @@ -494,7 +495,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchingCrossThreadDataThenCo const uint64_t baseAddress = 0x1000; auto &gfxCoreHelper = pClDevice->getGfxCoreHelper(); - auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment()); + const auto surfaceStateSize = static_cast(gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment())); auto patchValue1 = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(baseAddress)); auto patchValue2 = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(baseAddress + 1 * surfaceStateSize)); @@ -541,6 +542,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchBindlessSurfaceStatesInC pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.implicitArgs.globalConstantsSurfaceAddress.bindless = 4 * sizeof(uint64_t); MockKernel mockKernel(pProgram, pProgram->mockKernelInfo, *pClDevice); + mockKernel.kernelArguments.resize(pProgram->mockKernelInfo.kernelDescriptor.payloadMappings.explicitArgs.size()); pProgram->mockKernelInfo.kernelDescriptor.initBindlessOffsetToSurfaceState(); @@ -552,7 +554,7 @@ TEST_F(BindlessKernelTests, givenBindlessKernelWhenPatchBindlessSurfaceStatesInC ASSERT_TRUE(baseAddress > std::numeric_limits::max()); auto &gfxCoreHelper = pClDevice->getGfxCoreHelper(); - auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment()); + const auto surfaceStateSize = static_cast(gfxCoreHelper.getRenderSurfaceStateSize(pClDevice->getDevice().getRootDeviceEnvironment())); auto bindlessSufaceState1Address = baseAddress; auto bindlessSufaceState2Address = baseAddress + 2 * surfaceStateSize; @@ -3394,8 +3396,7 @@ HWTEST_F(KernelTest, givenBindlessArgBufferWhenPatchWithImplicitSurfaceThenSurfa kernel.mockKernel->patchWithImplicitSurface(castToUint64(&crossThreadData), mockAllocation, kernel.kernelInfo.argAsPtr(0)); const auto &gfxCoreHelper = device->getGfxCoreHelper(); - const auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(device->getRootDeviceEnvironment()); - + const auto surfaceStateSize = static_cast(gfxCoreHelper.getRenderSurfaceStateSize(device->getDevice().getRootDeviceEnvironment())); const auto ssIndex = kernel.kernelInfo.kernelDescriptor.bindlessArgsMap.find(bindlessOffset)->second; const auto ssOffset = ssIndex * surfaceStateSize; @@ -3437,6 +3438,8 @@ HWTEST_F(KernelTest, givenBindlessArgBufferAndNotInitializedBindlessOffsetToSurf } HWTEST2_F(KernelTest, givenBindlessHeapsHelperAndBindlessArgBufferWhenPatchWithImplicitSurfaceThenCrossThreadDataIsPatchedAndSurfaceStateIsEncoded, IsHeapfulRequired) { + DebugManagerStateRestore restore; + debugManager.flags.UseBindlessMode.set(1); auto device = clUniquePtr(new MockClDevice(MockDevice::createWithNewExecutionEnvironment(defaultHwInfo.get()))); auto &neoDevice = device->getDevice(); diff --git a/opencl/test/unit_test/os_interface/device_factory_tests.cpp b/opencl/test/unit_test/os_interface/device_factory_tests.cpp index 9adf278ea6843..eeff90d47abeb 100644 --- a/opencl/test/unit_test/os_interface/device_factory_tests.cpp +++ b/opencl/test/unit_test/os_interface/device_factory_tests.cpp @@ -19,6 +19,7 @@ #include "shared/test/common/mocks/mock_execution_environment.h" #include "shared/test/common/mocks/mock_io_functions.h" #include "shared/test/common/mocks/mock_memory_manager.h" +#include "shared/test/common/mocks/mock_memory_operations_handler.h" #include "shared/test/common/mocks/mock_product_helper.h" #include "shared/test/common/mocks/ult_device_factory.h" #include "shared/test/common/test_macros/test.h" @@ -273,6 +274,8 @@ TEST(DeviceFactory, givenCreateMultipleRootDevicesWhenCreateDevicesIsCalledThenV hwInfo[i] = *NEO::defaultHwInfo.get(); executionEnvironment->rootDeviceEnvironments[i]->setHwInfoAndInitHelpers(&hwInfo[i]); executionEnvironment->rootDeviceEnvironments[i]->initGmm(); + executionEnvironment->rootDeviceEnvironments[i]->memoryOperationsInterface = + std::make_unique(); } executionEnvironment->rootDeviceEnvironments[0]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; executionEnvironment->rootDeviceEnvironments[1]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; @@ -283,7 +286,9 @@ TEST(DeviceFactory, givenCreateMultipleRootDevicesWhenCreateDevicesIsCalledThenV executionEnvironment->rootDeviceEnvironments[6]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = true; executionEnvironment->rootDeviceEnvironments[7]->getMutableHardwareInfo()->capabilityTable.isIntegratedDevice = false; auto devices = DeviceFactory::createDevices(*executionEnvironment); - for (auto iterator = 0u; iterator < 8; iterator++) { + ASSERT_EQ(numRootDevices, devices.size()); + for (auto iterator = 0u; iterator < numRootDevices; iterator++) { + ASSERT_NE(nullptr, devices[iterator]); EXPECT_EQ(iterator, devices[iterator]->getRootDeviceIndex()); } } diff --git a/shared/source/command_stream/command_stream_receiver_hw.h b/shared/source/command_stream/command_stream_receiver_hw.h index 7a77544776dbe..cf95745b6a221 100644 --- a/shared/source/command_stream/command_stream_receiver_hw.h +++ b/shared/source/command_stream/command_stream_receiver_hw.h @@ -281,7 +281,8 @@ class CommandStreamReceiverHw : public CommandStreamReceiver { LinearStream &csrCommandStream, bool dispatchBindingTableCommand, bool areMultipleSubDevicesInContext, - bool setGeneralStateBaseAddress); + bool setGeneralStateBaseAddress, + bool useBindlessImages); inline void emitTagUpdateWithoutDCFlush(LinearStream &commandStream); @@ -372,6 +373,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver { CsrSizeRequestFlags csrSizeRequestFlags = {}; bool wasSubmittedToSingleSubdevice = false; + bool lastUseGlobalImageDescriptors = false; std::unique_ptr>> directSubmission; std::unique_ptr>> blitterDirectSubmission; diff --git a/shared/source/command_stream/command_stream_receiver_hw_base.inl b/shared/source/command_stream/command_stream_receiver_hw_base.inl index 968f1653d3252..afa6cd887b0d9 100644 --- a/shared/source/command_stream/command_stream_receiver_hw_base.inl +++ b/shared/source/command_stream/command_stream_receiver_hw_base.inl @@ -1686,7 +1686,10 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In bool stateBaseAddressDirty) { const auto bindlessHeapsHelper = device.getBindlessHeapsHelper(); - const bool useGlobalHeaps = bindlessHeapsHelper != nullptr; + + const bool useGlobalHeaps = bindlessHeapsHelper != nullptr && ApiSpecificConfig::getBindlessMode(device); + + const bool useGlobalImageDescriptors = bindlessHeapsHelper != nullptr && (useGlobalHeaps || dispatchFlags.useBindlessImages); auto &hwInfo = this->peekHwInfo(); @@ -1695,7 +1698,7 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In size_t dynamicStateSize = 0; if (hasDsh) { dynamicStateBaseAddress = NEO::getStateBaseAddress(*dsh, useGlobalHeaps); - dynamicStateSize = NEO::getStateSize(*dsh, bindlessHeapsHelper); + dynamicStateSize = NEO::getStateSize(*dsh, useGlobalHeaps); } int64_t surfaceStateBaseAddress = 0; @@ -1731,7 +1734,9 @@ inline void CommandStreamReceiverHw::programStateBaseAddress(const In surfaceStateBaseAddress, surfaceStateSize); } - bool isStateBaseAddressDirty = dshDirty || iohDirty || sshDirty || stateBaseAddressDirty; + const bool imageHeapSelectionChanged = useGlobalImageDescriptors != lastUseGlobalImageDescriptors; + bool isStateBaseAddressDirty = dshDirty || iohDirty || sshDirty || stateBaseAddressDirty || imageHeapSelectionChanged; + handleStateBaseAddressStateTransition(dispatchFlags, isStateBaseAddressDirty); // reprogram state base address command if required @@ -1782,7 +1787,8 @@ inline void CommandStreamReceiverHw::reprogramStateBaseAddress(const commandStreamCSR, bindingTableBaseAddressRequired, dispatchFlags.areMultipleSubDevicesInContext, - true); + true, + dispatchFlags.useBindlessImages); bindingTableBaseAddressRequired = false; setGSBAStateDirty(false); @@ -1802,7 +1808,8 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( LinearStream &csrCommandStream, bool dispatchBindingTableCommand, bool areMultipleSubDevicesInContext, - bool setGeneralStateBaseAddress) { + bool setGeneralStateBaseAddress, + bool useBindlessImages) { using STATE_BASE_ADDRESS = typename GfxFamily::STATE_BASE_ADDRESS; auto &rootDeviceEnvironment = this->peekRootDeviceEnvironment(); @@ -1817,12 +1824,22 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( auto globalHeapsBase = 0ull; bool useGlobalSshAndDsh = false; - if (device.getBindlessHeapsHelper()) { - bindlessSurfStateBase = device.getBindlessHeapsHelper()->getGlobalHeapsBase(); - globalHeapsBase = device.getBindlessHeapsHelper()->getGlobalHeapsBase(); + + auto *helper = device.getBindlessHeapsHelper(); + + const bool useGlobalHeaps = helper != nullptr && ApiSpecificConfig::getBindlessMode(device); + + const bool useGlobalImageDescriptors = helper != nullptr && (useGlobalHeaps || useBindlessImages); + + if (useGlobalHeaps) { + globalHeapsBase = helper->getGlobalHeapsBase(); useGlobalSshAndDsh = true; } + if (useGlobalImageDescriptors) { + bindlessSurfStateBase = helper->getGlobalHeapsBase(); + } + STATE_BASE_ADDRESS stateBaseAddressCmd; StateBaseAddressHelperArgs args = { generalStateBaseAddress, // generalStateBaseAddress @@ -1854,6 +1871,8 @@ inline void CommandStreamReceiverHw::programStateBaseAddressCommon( StateBaseAddressHelper::programStateBaseAddressIntoCommandStream(args, csrCommandStream); + lastUseGlobalImageDescriptors = useGlobalImageDescriptors; + bool sbaTrackingEnabled = debuggingEnabled; if (sbaTrackingEnabled) { device.getL0Debugger()->programSbaAddressLoad(csrCommandStream, @@ -2178,7 +2197,7 @@ void CommandStreamReceiverHw::dispatchImmediateFlushStateBaseAddressC if (flushData.stateBaseAddressDirty) { bool btCommandNeeded = this->streamProperties.stateBaseAddress.bindingTablePoolBaseAddress.value != StreamProperty64::initValue; programStateBaseAddressCommon(nullptr, nullptr, nullptr, &this->streamProperties.stateBaseAddress, - 0, 0, flushData.pipelineSelectArgs, device, csrStream, btCommandNeeded, device.getNumGenericSubDevices() > 1, false); + 0, 0, flushData.pipelineSelectArgs, device, csrStream, btCommandNeeded, device.getNumGenericSubDevices() > 1, false,false); this->streamProperties.stateBaseAddress.clearIsDirty(); } } diff --git a/shared/source/command_stream/csr_definitions.h b/shared/source/command_stream/csr_definitions.h index 5a2b141786d1b..39d93e96d0ac0 100644 --- a/shared/source/command_stream/csr_definitions.h +++ b/shared/source/command_stream/csr_definitions.h @@ -126,6 +126,7 @@ struct DispatchFlags { bool isStallingCommandsOnNextFlushRequired = false; bool isDcFlushRequiredOnStallingCommandsOnNextFlush = false; bool isWalkerWithProfilingEnqueued = false; + bool useBindlessImages = false; }; struct CsrSizeRequestFlags { diff --git a/shared/source/compiler_interface/compiler_options.h b/shared/source/compiler_interface/compiler_options.h index 619019eab155e..0d834316661a0 100644 --- a/shared/source/compiler_interface/compiler_options.h +++ b/shared/source/compiler_interface/compiler_options.h @@ -31,6 +31,8 @@ inline constexpr ConstStringRef createLibrary = "-create-library"; inline constexpr ConstStringRef generateDebugInfo = "-g"; inline constexpr ConstStringRef generateSourcePath = "-s"; inline constexpr ConstStringRef bindlessMode = "-cl-intel-use-bindless-mode -cl-intel-use-bindless-advanced-mode"; +inline constexpr ConstStringRef bindlessImages = "-cl-intel-use-bindless-images"; +inline constexpr ConstStringRef bindlessAdvancedMode = "-cl-intel-use-bindless-advanced-mode"; inline constexpr ConstStringRef uniformWorkgroupSize = "-cl-uniform-work-group-size"; inline constexpr ConstStringRef forceEmuInt32DivRem = "-cl-intel-force-emu-int32divrem"; inline constexpr ConstStringRef forceEmuInt32DivRemSP = "-cl-intel-force-emu-sp-int32divrem"; diff --git a/shared/source/device/root_device.cpp b/shared/source/device/root_device.cpp index c7c51e757e2a0..d97b3a16bdf22 100644 --- a/shared/source/device/root_device.cpp +++ b/shared/source/device/root_device.cpp @@ -51,8 +51,10 @@ Device *RootDevice::getRootDevice() const { void RootDevice::createBindlessHeapsHelper() { - if (!NEO::debugManager.flags.NEO_L0_SYSMAN_NO_CONTEXT_MODE.get() && ApiSpecificConfig::getGlobalBindlessHeapConfiguration(this->getHardwareInfo()) && ApiSpecificConfig::getBindlessMode(*this)) { - this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper(this, getNumGenericSubDevices() > 1); + if (!NEO::debugManager.flags.NEO_L0_SYSMAN_NO_CONTEXT_MODE.get() && + ApiSpecificConfig::createBindlessHeapsHelper(*this)) { + this->executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->createBindlessHeapsHelper( + this, getNumGenericSubDevices() > 1); } } diff --git a/shared/source/execution_environment/root_device_environment.cpp b/shared/source/execution_environment/root_device_environment.cpp index a094fe36f4223..b8371aa0f17f2 100644 --- a/shared/source/execution_environment/root_device_environment.cpp +++ b/shared/source/execution_environment/root_device_environment.cpp @@ -157,7 +157,11 @@ const ProductHelper &RootDeviceEnvironment::getProductHelper() const { } void RootDeviceEnvironment::createBindlessHeapsHelper(Device *rootDevice, bool availableDevices) { - bindlessHeapsHelper = std::make_unique(rootDevice, availableDevices); + std::lock_guard lock(bindlessHeapsHelperMutex); + + if (bindlessHeapsHelper == nullptr) { + bindlessHeapsHelper = std::make_unique(rootDevice, availableDevices); + } } FrontEndController *RootDeviceEnvironment::getFrontEndController() const { diff --git a/shared/source/execution_environment/root_device_environment.h b/shared/source/execution_environment/root_device_environment.h index add91ad6bdc7d..3818bc8182033 100644 --- a/shared/source/execution_environment/root_device_environment.h +++ b/shared/source/execution_environment/root_device_environment.h @@ -159,6 +159,7 @@ struct RootDeviceEnvironment : NonCopyableClass { private: std::mutex mtx; + std::mutex bindlessHeapsHelperMutex; }; static_assert(NEO::NonCopyable); diff --git a/shared/source/gmm_helper/cache_settings_helper.cpp b/shared/source/gmm_helper/cache_settings_helper.cpp index 2fa8738a213c1..f6b0a6e488165 100644 --- a/shared/source/gmm_helper/cache_settings_helper.cpp +++ b/shared/source/gmm_helper/cache_settings_helper.cpp @@ -94,6 +94,7 @@ GmmResourceUsageType CacheSettingsHelper::getDefaultUsageTypeWithCachingEnabled( return GMM_RESOURCE_USAGE_OCL_IMAGE; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: if (debugManager.flags.DisableCachingForHeaps.get()) { return getDefaultUsageTypeWithCachingDisabled(allocationType, productHelper); } @@ -142,6 +143,7 @@ GmmResourceUsageType CacheSettingsHelper::getDefaultUsageTypeWithCachingDisabled return GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: return GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED; default: return productHelper.isNewCoherencyModelSupported() ? GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC : GMM_RESOURCE_USAGE_OCL_BUFFER_CACHELINE_MISALIGNED; diff --git a/shared/source/helpers/api_specific_config.h b/shared/source/helpers/api_specific_config.h index 7441e077bf375..9f7ed3a4de792 100644 --- a/shared/source/helpers/api_specific_config.h +++ b/shared/source/helpers/api_specific_config.h @@ -44,5 +44,6 @@ struct ApiSpecificConfig { static std::string compilerCacheFileExtension(); static int64_t compilerCacheDefaultEnabled(); static bool isUpdateTagFromWaitEnabledForHeapless(); + static bool createBindlessHeapsHelper(const Device &device); }; } // namespace NEO diff --git a/shared/source/helpers/app_resource_classification.h b/shared/source/helpers/app_resource_classification.h index b529050c69c1e..bdb91fb6c1347 100644 --- a/shared/source/helpers/app_resource_classification.h +++ b/shared/source/helpers/app_resource_classification.h @@ -51,6 +51,7 @@ class AppResourceClassification { case AllocationType::kernelArgsBuffer: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::mcs: case AllocationType::preemption: case AllocationType::profilingTagBuffer: diff --git a/shared/source/helpers/app_resource_helper.cpp b/shared/source/helpers/app_resource_helper.cpp index 022107821ecef..011ddd78f4d33 100644 --- a/shared/source/helpers/app_resource_helper.cpp +++ b/shared/source/helpers/app_resource_helper.cpp @@ -56,6 +56,8 @@ const char *AppResourceHelper::getResourceTagStr(AllocationType type) { return "KRLISAIN"; case AllocationType::linearStream: return "LINRSTRM"; + case AllocationType::bindlessHeap: + return "BNDLHEAP"; case AllocationType::mapAllocation: return "MAPALLOC"; case AllocationType::mcs: diff --git a/shared/source/helpers/bindless_heaps_helper.cpp b/shared/source/helpers/bindless_heaps_helper.cpp index a0d48ec0922a4..a721440ae4cf8 100644 --- a/shared/source/helpers/bindless_heaps_helper.cpp +++ b/shared/source/helpers/bindless_heaps_helper.cpp @@ -11,6 +11,7 @@ #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/compiler_product_helper.h" #include "shared/source/helpers/driver_model_type.h" #include "shared/source/helpers/gfx_core_helper.h" @@ -66,13 +67,14 @@ BindlessHeapsHelper::BindlessHeapsHelper(Device *rootDevice, bool isMultiOsConte } } - auto heapAllocation = getHeapAllocation(size, MemoryConstants::pageSize64k, heapType == BindlesHeapType::specialSsh); + auto heapAllocation = getHeapAllocation(size, MemoryConstants::pageSize64k, heapType == BindlesHeapType::specialSsh, + static_cast(heapType)); UNRECOVERABLE_IF(heapAllocation == nullptr); ssHeapsAllocations.push_back(heapAllocation); surfaceStateHeaps[heapType] = std::make_unique(heapAllocation, true); } - borderColorStates = getHeapAllocation(MemoryConstants::pageSize, MemoryConstants::pageSize, false); + borderColorStates = getHeapAllocation(MemoryConstants::pageSize, MemoryConstants::pageSize, false, BindlesHeapType::specialSsh); UNRECOVERABLE_IF(borderColorStates == nullptr); float borderColorDefault[4] = {0, 0, 0, 0}; memcpy_s(borderColorStates->getUnderlyingBuffer(), sizeof(borderColorDefault), borderColorDefault, sizeof(borderColorDefault)); @@ -107,6 +109,15 @@ bool BindlessHeapsHelper::tryReservingMemoryForSpecialSsh(const size_t size, siz return reservedRange.has_value(); } +AllocationType BindlessHeapsHelper::getHeapAllocationType(BindlesHeapType heapType) const { + if (ApiSpecificConfig::getApiType() == ApiSpecificConfig::OCL && + heapType == BindlesHeapType::globalSsh) { + return AllocationType::bindlessHeap; + } + + return AllocationType::linearStream; +} + bool BindlessHeapsHelper::initializeReservedMemory() { if (reservedMemoryInitialized) { return true; @@ -127,8 +138,8 @@ bool BindlessHeapsHelper::initializeReservedMemory() { heapFrontWindow = std::make_unique(reservedRangeBase, heapFrontWindowSize, MemoryConstants::pageSize64k, 0); heapRegular = std::make_unique(reservedRangeBase + heapFrontWindowSize, heapRegularSize, MemoryConstants::pageSize64k, 0); - memManager->addCustomHeapAllocatorConfig(AllocationType::linearStream, true, rootDeviceIndex, {heapFrontWindow.get(), reservedRangeBase}); - memManager->addCustomHeapAllocatorConfig(AllocationType::linearStream, false, rootDeviceIndex, {heapRegular.get(), reservedRangeBase}); + memManager->addCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), true, rootDeviceIndex, {heapFrontWindow.get(), reservedRangeBase}); + memManager->addCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), false, rootDeviceIndex, {heapRegular.get(), reservedRangeBase}); reservedMemoryInitialized = true; return true; @@ -147,13 +158,13 @@ BindlessHeapsHelper::~BindlessHeapsHelper() { reservedRanges.clear(); if (reservedMemoryInitialized) { - memManager->removeCustomHeapAllocatorConfig(AllocationType::linearStream, true, rootDeviceIndex); - memManager->removeCustomHeapAllocatorConfig(AllocationType::linearStream, false, rootDeviceIndex); + memManager->removeCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), true, rootDeviceIndex); + memManager->removeCustomHeapAllocatorConfig(getHeapAllocationType(BindlesHeapType::specialSsh), false, rootDeviceIndex); } } -GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow) { - auto allocationType = AllocationType::linearStream; +GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow, BindlesHeapType heapType) { + auto allocationType = getHeapAllocationType(heapType); NEO::AllocationProperties properties{rootDeviceIndex, true, heapSize, allocationType, isMultiOsContextCapable, deviceBitfield}; properties.flags.use32BitFrontWindow = allocInFrontWindow; properties.alignment = alignment; @@ -280,7 +291,7 @@ IndirectHeap *BindlessHeapsHelper::getHeap(BindlesHeapType heapType) { bool BindlessHeapsHelper::growHeap(BindlesHeapType heapType) { auto heap = surfaceStateHeaps[heapType].get(); auto allocInFrontWindow = false; - auto newAlloc = getHeapAllocation(globalSshAllocationSize, MemoryConstants::pageSize64k, allocInFrontWindow); + auto newAlloc = getHeapAllocation(globalSshAllocationSize, MemoryConstants::pageSize64k, allocInFrontWindow, heapType); DEBUG_BREAK_IF(newAlloc == nullptr); if (newAlloc == nullptr) { return false; diff --git a/shared/source/helpers/bindless_heaps_helper.h b/shared/source/helpers/bindless_heaps_helper.h index 0eb806c60d28b..1ed9b6f67e173 100644 --- a/shared/source/helpers/bindless_heaps_helper.h +++ b/shared/source/helpers/bindless_heaps_helper.h @@ -44,7 +44,7 @@ class BindlessHeapsHelper : NEO::NonCopyableAndNonMovableClass { BindlessHeapsHelper(Device *rootDevice, bool isMultiOsContextCapable); MOCKABLE_VIRTUAL ~BindlessHeapsHelper(); - GraphicsAllocation *getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow); + GraphicsAllocation *getHeapAllocation(size_t heapSize, size_t alignment, bool allocInFrontWindow, BindlesHeapType heapType); MOCKABLE_VIRTUAL SurfaceStateInHeapInfo allocateSSInHeap(size_t ssSize, GraphicsAllocation *surfaceAllocation, BindlesHeapType heapType); uint64_t getGlobalHeapsBase(); @@ -74,6 +74,7 @@ class BindlessHeapsHelper : NEO::NonCopyableAndNonMovableClass { bool initializeReservedMemory(); bool isReservedMemoryModeAvailable(); bool tryReservingMemoryForSpecialSsh(const size_t size, size_t alignment); + AllocationType getHeapAllocationType(BindlesHeapType heapType) const; std::optional reserveMemoryRange(size_t size, size_t alignment, HeapIndex heapIndex); std::mutex mtx; diff --git a/shared/source/helpers/heap_assigner.cpp b/shared/source/helpers/heap_assigner.cpp index 847d1ea8a06dd..23dbbaa55ecdf 100644 --- a/shared/source/helpers/heap_assigner.cpp +++ b/shared/source/helpers/heap_assigner.cpp @@ -30,6 +30,9 @@ HeapIndex HeapAssigner::get32BitHeapIndex(AllocationType allocType, bool useLoca return useFrontWindow ? mapExternalWindowIndex(MemoryManager::selectExternalHeap(useLocalMem)) : MemoryManager::selectExternalHeap(useLocalMem); } bool HeapAssigner::useExternal32BitHeap(AllocationType allocType) { + if (allocType == AllocationType::bindlessHeap) { + return true; + } if (apiAllowExternalHeapForSshAndDsh) { return allocType == AllocationType::linearStream; } diff --git a/shared/source/helpers/memory_properties_flags.h b/shared/source/helpers/memory_properties_flags.h index 5611a3c206d64..4d2c03667501d 100644 --- a/shared/source/helpers/memory_properties_flags.h +++ b/shared/source/helpers/memory_properties_flags.h @@ -35,6 +35,7 @@ struct MemoryFlags { uint32_t compressedHint : 1; uint32_t uncompressedHint : 1; uint32_t ipcSupportedAllocationByDefault : 1; + uint32_t bindlessImage : 1; bool operator==(const MemoryFlags &) const = default; }; diff --git a/shared/source/memory_manager/allocation_type.h b/shared/source/memory_manager/allocation_type.h index 0587ce35fd472..f68099a2cb2e4 100644 --- a/shared/source/memory_manager/allocation_type.h +++ b/shared/source/memory_manager/allocation_type.h @@ -28,6 +28,7 @@ enum class AllocationType { kernelIsa, kernelIsaInternal, linearStream, + bindlessHeap, mapAllocation, mcs, preemption, diff --git a/shared/source/memory_manager/definitions/storage_info.cpp b/shared/source/memory_manager/definitions/storage_info.cpp index ee31c4a26c178..12c35ef613182 100644 --- a/shared/source/memory_manager/definitions/storage_info.cpp +++ b/shared/source/memory_manager/definitions/storage_info.cpp @@ -97,6 +97,7 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope case AllocationType::commandBuffer: case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::syncBuffer: storageInfo.cloningOfPageTables = properties.flags.multiOsContextCapable; if (!properties.flags.multiOsContextCapable) { @@ -232,6 +233,7 @@ DeviceBitfield MemoryManager::computeStorageInfoMemoryBanks(const AllocationProp case AllocationType::commandBuffer: case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: memoryBanks = preferredBank; break; case AllocationType::scratchSurface: diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index bd031c491039f..ccda49c9d27aa 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -444,6 +444,7 @@ class GraphicsAllocation : public IDNode, NEO::NonCopyableAn case AllocationType::kernelIsa: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::ringBuffer: return false; default: diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index c8ddf67a906ad..c9c87df19a49d 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -699,6 +699,7 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo case AllocationType::kernelIsa: case AllocationType::kernelIsaInternal: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::mcs: case AllocationType::preemption: case AllocationType::scratchSurface: @@ -1384,6 +1385,7 @@ bool MemoryManager::isAllocationTypeToCapture(AllocationType type) const { case AllocationType::scratchSurface: case AllocationType::privateSurface: case AllocationType::linearStream: + case AllocationType::bindlessHeap: case AllocationType::internalHeap: return true; default: diff --git a/shared/source/utilities/logger.cpp b/shared/source/utilities/logger.cpp index 2abeaca37f2ee..02fc5a5afa969 100644 --- a/shared/source/utilities/logger.cpp +++ b/shared/source/utilities/logger.cpp @@ -313,6 +313,8 @@ const char *getAllocationTypeString(GraphicsAllocation const *graphicsAllocation return "KERNEL_ISA_INTERNAL"; case AllocationType::linearStream: return "LINEAR_STREAM"; + case AllocationType::bindlessHeap: + return "BINDLESS_HEAP"; case AllocationType::mapAllocation: return "MAP_ALLOCATION"; case AllocationType::mcs: diff --git a/shared/test/unit_test/api_specific_config_ult.cpp b/shared/test/unit_test/api_specific_config_ult.cpp index 15fb234eed7c3..587947f638de1 100644 --- a/shared/test/unit_test/api_specific_config_ult.cpp +++ b/shared/test/unit_test/api_specific_config_ult.cpp @@ -7,7 +7,7 @@ #include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/helpers/api_specific_config.h" - +#include "shared/source/device/device.h" #include #include @@ -130,5 +130,9 @@ bool ApiSpecificConfig::isGlobalStatelessEnabled(const RootDeviceEnvironment &ro return globalStatelessOcl; } } +bool ApiSpecificConfig::createBindlessHeapsHelper(const Device &device) { + return getGlobalBindlessHeapConfiguration(device.getHardwareInfo()) && + getBindlessMode(device); +} } // namespace NEO diff --git a/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp b/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp index 40b7425088efe..a925fd6d58f47 100644 --- a/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp +++ b/shared/test/unit_test/gmm_helper/gmm_helper_tests.cpp @@ -769,6 +769,7 @@ TEST(GmmTest, givenAllocationTypeWhenGettingUsageTypeThenReturnCorrectValue) { break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = forceUncached ? GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED : GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::gpuTimestampDeviceBuffer: @@ -818,6 +819,7 @@ TEST(GmmTest, givenAllocationTypeWhenGettingUsageTypeThenReturnCorrectValue) { break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = forceUncached ? GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED : GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::gpuTimestampDeviceBuffer: @@ -888,6 +890,7 @@ TEST(GmmTest, givenAllocationTypeAndMitigatedDcFlushWhenGettingUsageTypeThenRetu break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::bufferHostMemory: @@ -926,6 +929,7 @@ TEST(GmmTest, givenAllocationTypeAndMitigatedDcFlushWhenGettingUsageTypeThenRetu break; case AllocationType::internalHeap: case AllocationType::linearStream: + case AllocationType::bindlessHeap: expectedUsage = GMM_RESOURCE_USAGE_OCL_STATE_HEAP_BUFFER; break; case AllocationType::bufferHostMemory: @@ -1001,7 +1005,9 @@ TEST(GmmTest, givenForceAllResourcesUncachedFlagSetWhenGettingUsageTypeThenRetur if (allocationType == AllocationType::preemption) { expectedUsage = GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC; - } else if ((allocationType == AllocationType::internalHeap) || (allocationType == AllocationType::linearStream)) { + } else if ((allocationType == AllocationType::internalHeap) || + (allocationType == AllocationType::linearStream) || + (allocationType == AllocationType::bindlessHeap)) { expectedUsage = GMM_RESOURCE_USAGE_OCL_SYSTEM_MEMORY_BUFFER_CACHELINE_MISALIGNED; } diff --git a/shared/test/unit_test/helpers/app_resource_classification_tests.cpp b/shared/test/unit_test/helpers/app_resource_classification_tests.cpp index 2f82b2830b980..7d7d0809f5a58 100644 --- a/shared/test/unit_test/helpers/app_resource_classification_tests.cpp +++ b/shared/test/unit_test/helpers/app_resource_classification_tests.cpp @@ -44,6 +44,7 @@ TEST(AppResourceClassificationTest, givenInternalAllocationTypesWhenCheckingIsAp EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::kernelArgsBuffer)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::kernelIsaInternal)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::linearStream)); + EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::bindlessHeap)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::mcs)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::preemption)); EXPECT_FALSE(AppResourceClassification::isApplicationResource(AllocationType::profilingTagBuffer)); @@ -68,7 +69,7 @@ TEST(AppResourceClassificationTest, givenInternalAllocationTypesWhenCheckingIsAp TEST(AppResourceClassificationTest, givenAllAllocationTypesWhenCheckingIsApplicationResourceThenAllTypesAreCovered) { constexpr int expectedApplicationTypes = 20; - constexpr int expectedDriverTypes = 29; + constexpr int expectedDriverTypes = 30; int applicationTypesCount = 0; int driverTypesCount = 0; diff --git a/shared/test/unit_test/helpers/app_resource_tests.cpp b/shared/test/unit_test/helpers/app_resource_tests.cpp index 78ecbb3746f8a..db17312f34cfc 100644 --- a/shared/test/unit_test/helpers/app_resource_tests.cpp +++ b/shared/test/unit_test/helpers/app_resource_tests.cpp @@ -71,6 +71,7 @@ AllocationTypeTagTestCase allocationTypeTagValues[static_cast(AllocationTyp {AllocationType::kernelIsa, "KERNLISA"}, {AllocationType::kernelIsaInternal, "KRLISAIN"}, {AllocationType::linearStream, "LINRSTRM"}, + {AllocationType::bindlessHeap, "BNDLHEAP"}, {AllocationType::mapAllocation, "MAPALLOC"}, {AllocationType::mcs, "MCS"}, {AllocationType::preemption, "PRMPTION"}, diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp index 80c54f764cdf7..ccb1a204a0a7e 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_bindless_heap_tests.cpp @@ -7,6 +7,7 @@ #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/bindless_heaps_helper.h" #include "shared/source/indirect_heap/indirect_heap.h" #include "shared/source/os_interface/device_factory.h" @@ -20,16 +21,22 @@ #include "shared/test/common/test_macros/test.h" namespace NEO { +extern ApiSpecificConfig::ApiType apiTypeForUlts; + struct GlobalBindlessDrmMemManagerFixture : public DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { GlobalBindlessDrmMemManagerFixture() : DrmMemoryManagerFixtureWithoutQuietIoctlExpectation(1, 0) {} void setUp() { + previousApiType = apiTypeForUlts; + apiTypeForUlts = ApiSpecificConfig::L0; debugManager.flags.UseExternalAllocatorForSshAndDsh.set(true); DrmMemoryManagerFixtureWithoutQuietIoctlExpectation::setUp(true); } void tearDown() { + apiTypeForUlts = previousApiType; } DebugManagerStateRestore dbgRestorer; + ApiSpecificConfig::ApiType previousApiType = ApiSpecificConfig::OCL; }; using DrmGlobalBindlessAllocatorTests = Test;