Organize dispatching of thread data for better reuse of code

Change-Id: I8c156f8b5a50f6fa4dfb5218cdadb2840ff556eb

Author: zzdanowicz

Jenkinsfile

@@ -1,5 +1,5 @@
 #!groovy
 neoDependenciesRev='803657-1097'
 strategy='EQUAL'
-allowedCD=274
+allowedCD=273
 allowedF=4

runtime/command_queue/CMakeLists.txt

@@ -36,6 +36,7 @@ set(RUNTIME_SRCS_COMMAND_QUEUE
   ${CMAKE_CURRENT_SOURCE_DIR}/gpgpu_walker.inl
   ${CMAKE_CURRENT_SOURCE_DIR}/hardware_interface.h
   ${CMAKE_CURRENT_SOURCE_DIR}/hardware_interface.inl
+  ${CMAKE_CURRENT_SOURCE_DIR}/hardware_interface_base.inl
   ${CMAKE_CURRENT_SOURCE_DIR}/local_id_gen.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/local_id_gen.h
   ${CMAKE_CURRENT_SOURCE_DIR}/local_id_gen.inl

runtime/command_queue/gpgpu_walker.h

@@ -26,8 +26,6 @@
 
 namespace OCLRT {
 
-using WALKER_HANDLE = void *;
-
 template <typename GfxFamily>
 using WALKER_TYPE = typename GfxFamily::WALKER_TYPE;
 
@@ -129,7 +127,7 @@ class GpgpuWalkerHelper {
     static size_t getSizeForWADisableLSQCROPERFforOCL(const Kernel *pKernel);
 
     static size_t setGpgpuWalkerThreadData(
-        WALKER_HANDLE pCmdData,
+        WALKER_TYPE<GfxFamily> *walkerCmd,
         const size_t globalOffsets[3],
         const size_t startWorkGroups[3],
         const size_t numWorkGroups[3],
@@ -187,7 +185,7 @@ class GpgpuWalkerHelper {
 
     static void setupTimestampPacket(
         LinearStream *cmdStream,
-        WALKER_HANDLE walkerHandle,
+        WALKER_TYPE<GfxFamily> *walkerCmd,
         TimestampPacket *timestampPacket,
         TimestampPacket::WriteOperationType writeOperationType);
 

runtime/command_queue/gpgpu_walker.inl

@@ -101,24 +101,22 @@ void GpgpuWalkerHelper<GfxFamily>::addAluReadModifyWriteRegister(
 
 template <typename GfxFamily>
 inline size_t GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(
-    WALKER_HANDLE pCmdData,
+    WALKER_TYPE<GfxFamily> *walkerCmd,
     const size_t globalOffsets[3],
     const size_t startWorkGroups[3],
     const size_t numWorkGroups[3],
     const size_t localWorkSizesIn[3],
     uint32_t simd,
     uint32_t workDim,
     bool localIdsGeneration) {
-    WALKER_TYPE<GfxFamily> *pCmd = static_cast<WALKER_TYPE<GfxFamily> *>(pCmdData);
-
     auto localWorkSize = localWorkSizesIn[0] * localWorkSizesIn[1] * localWorkSizesIn[2];
 
     auto threadsPerWorkGroup = getThreadsPerWG(simd, localWorkSize);
-    pCmd->setThreadWidthCounterMaximum(static_cast<uint32_t>(threadsPerWorkGroup));
+    walkerCmd->setThreadWidthCounterMaximum(static_cast<uint32_t>(threadsPerWorkGroup));
 
-    pCmd->setThreadGroupIdXDimension(static_cast<uint32_t>(numWorkGroups[0]));
-    pCmd->setThreadGroupIdYDimension(static_cast<uint32_t>(numWorkGroups[1]));
-    pCmd->setThreadGroupIdZDimension(static_cast<uint32_t>(numWorkGroups[2]));
+    walkerCmd->setThreadGroupIdXDimension(static_cast<uint32_t>(numWorkGroups[0]));
+    walkerCmd->setThreadGroupIdYDimension(static_cast<uint32_t>(numWorkGroups[1]));
+    walkerCmd->setThreadGroupIdZDimension(static_cast<uint32_t>(numWorkGroups[2]));
 
     // compute executionMask - to tell which SIMD lines are active within thread
     auto remainderSimdLanes = localWorkSize & (simd - 1);
@@ -128,13 +126,13 @@ inline size_t GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(
 
     using SIMD_SIZE = typename WALKER_TYPE<GfxFamily>::SIMD_SIZE;
 
-    pCmd->setRightExecutionMask(static_cast<uint32_t>(executionMask));
-    pCmd->setBottomExecutionMask(static_cast<uint32_t>(0xffffffff));
-    pCmd->setSimdSize(static_cast<SIMD_SIZE>(simd >> 4));
+    walkerCmd->setRightExecutionMask(static_cast<uint32_t>(executionMask));
+    walkerCmd->setBottomExecutionMask(static_cast<uint32_t>(0xffffffff));
+    walkerCmd->setSimdSize(static_cast<SIMD_SIZE>(simd >> 4));
 
-    pCmd->setThreadGroupIdStartingX(static_cast<uint32_t>(startWorkGroups[0]));
-    pCmd->setThreadGroupIdStartingY(static_cast<uint32_t>(startWorkGroups[1]));
-    pCmd->setThreadGroupIdStartingResumeZ(static_cast<uint32_t>(startWorkGroups[2]));
+    walkerCmd->setThreadGroupIdStartingX(static_cast<uint32_t>(startWorkGroups[0]));
+    walkerCmd->setThreadGroupIdStartingY(static_cast<uint32_t>(startWorkGroups[1]));
+    walkerCmd->setThreadGroupIdStartingResumeZ(static_cast<uint32_t>(startWorkGroups[2]));
 
     return localWorkSize;
 }
@@ -432,7 +430,7 @@ inline void GpgpuWalkerHelper<GfxFamily>::dispatchOnDeviceWaitlistSemaphores(Lin
 template <typename GfxFamily>
 void GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(
     LinearStream *cmdStream,
-    WALKER_HANDLE walkerHandle,
+    WALKER_TYPE<GfxFamily> *walkerCmd,
     TimestampPacket *timestampPacket,
     TimestampPacket::WriteOperationType writeOperationType) {
 
@@ -523,8 +521,12 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchScheduler(
     indirectObjectHeap.getSpace(curbeOffset);
     ioh = &indirectObjectHeap;
 
-    bool localIdsGeneration = KernelCommandsHelper<GfxFamily>::isDispatchForLocalIdsGeneration(1, globalWorkSizes, localWorkSizes);
-    auto offsetCrossThreadData = KernelCommandsHelper<GfxFamily>::sendIndirectState(
+    // Program the walker.  Invokes execution so all state should already be programmed
+    auto pGpGpuWalkerCmd = (GPGPU_WALKER *)commandStream->getSpace(sizeof(GPGPU_WALKER));
+    *pGpGpuWalkerCmd = GfxFamily::cmdInitGpgpuWalker;
+
+    bool localIdsGeneration = KernelCommandsHelper<GfxFamily>::isRuntimeLocalIdsGenerationRequired(1, globalWorkSizes, localWorkSizes);
+    KernelCommandsHelper<GfxFamily>::sendIndirectState(
         *commandStream,
         *dsh,
         *ioh,
@@ -535,37 +537,16 @@ void GpgpuWalkerHelper<GfxFamily>::dispatchScheduler(
         offsetInterfaceDescriptorTable,
         interfaceDescriptorIndex,
         preemptionMode,
+        pGpGpuWalkerCmd,
         nullptr,
         localIdsGeneration);
 
     // Implement enabling special WA DisableLSQCROPERFforOCL if needed
     GpgpuWalkerHelper<GfxFamily>::applyWADisableLSQCROPERFforOCL(commandStream, scheduler, true);
 
-    // Program the walker.  Invokes execution so all state should already be programmed
-    auto pGpGpuWalkerCmd = (GPGPU_WALKER *)commandStream->getSpace(sizeof(GPGPU_WALKER));
-    *pGpGpuWalkerCmd = GfxFamily::cmdInitGpgpuWalker;
-
     size_t globalOffsets[3] = {0, 0, 0};
     size_t workGroups[3] = {(scheduler.getGws() / scheduler.getLws()), 1, 1};
-    auto localWorkSize = GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(pGpGpuWalkerCmd, globalOffsets, globalOffsets, workGroups, localWorkSizes, simd, 1, localIdsGeneration);
-
-    pGpGpuWalkerCmd->setIndirectDataStartAddress((uint32_t)offsetCrossThreadData);
-    DEBUG_BREAK_IF(offsetCrossThreadData % 64 != 0);
-    pGpGpuWalkerCmd->setInterfaceDescriptorOffset(interfaceDescriptorIndex);
-
-    auto threadPayload = scheduler.getKernelInfo().patchInfo.threadPayload;
-    DEBUG_BREAK_IF(nullptr == threadPayload);
-
-    auto numChannels = PerThreadDataHelper::getNumLocalIdChannels(*threadPayload);
-    auto localIdSizePerThread = PerThreadDataHelper::getLocalIdSizePerThread(simd, numChannels);
-    localIdSizePerThread = std::max(localIdSizePerThread, sizeof(GRF));
-
-    auto sizePerThreadDataTotal = getThreadsPerWG(simd, localWorkSize) * localIdSizePerThread;
-    DEBUG_BREAK_IF(sizePerThreadDataTotal == 0); // Hardware requires at least 1 GRF of perThreadData for each thread in thread group
-
-    auto sizeCrossThreadData = scheduler.getCrossThreadDataSize();
-    auto IndirectDataLength = alignUp((uint32_t)(sizeCrossThreadData + sizePerThreadDataTotal), GPGPU_WALKER::INDIRECTDATASTARTADDRESS_ALIGN_SIZE);
-    pGpGpuWalkerCmd->setIndirectDataLength(IndirectDataLength);
+    GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(pGpGpuWalkerCmd, globalOffsets, globalOffsets, workGroups, localWorkSizes, simd, 1, localIdsGeneration);
 
     // Implement disabling special WA DisableLSQCROPERFforOCL if needed
     GpgpuWalkerHelper<GfxFamily>::applyWADisableLSQCROPERFforOCL(commandStream, scheduler, false);

runtime/command_queue/hardware_interface.h

@@ -26,7 +26,8 @@ struct MultiDispatchInfo;
 template <class T>
 struct TagNode;
 
-using WALKER_HANDLE = void *;
+template <typename GfxFamily>
+using WALKER_TYPE = typename GfxFamily::WALKER_TYPE;
 
 template <typename GfxFamily>
 class HardwareInterface {
@@ -57,12 +58,7 @@ class HardwareInterface {
         LinearStream *commandStream);
 
     static INTERFACE_DESCRIPTOR_DATA *obtainInterfaceDescriptorData(
-        WALKER_HANDLE pCmdData);
-
-    static void setOffsetCrossThreadData(
-        WALKER_HANDLE pCmdData,
-        size_t &offsetCrossThreadData,
-        uint32_t &interfaceDescriptorIndex);
+        WALKER_TYPE<GfxFamily> *walkerCmd);
 
     static void dispatchWorkarounds(
         LinearStream *commandStream,
@@ -83,6 +79,9 @@ class HardwareInterface {
         HwPerfCounter *hwPerfCounter,
         LinearStream *commandStream,
         CommandQueue &commandQueue);
+
+    static WALKER_TYPE<GfxFamily> *allocateWalkerSpace(LinearStream &commandStream,
+                                                       const Kernel &kernel);
 };
 
 } // namespace OCLRT

runtime/command_queue/hardware_interface.inl

@@ -188,18 +188,17 @@ void HardwareInterface<GfxFamily>::dispatchWalker(
         }
 
         // Program the walker.  Invokes execution so all state should already be programmed
-        auto pWalkerCmd = static_cast<WALKER_TYPE<GfxFamily> *>(commandStream->getSpace(sizeof(WALKER_TYPE<GfxFamily>)));
-        *pWalkerCmd = GfxFamily::cmdInitGpgpuWalker;
+        auto walkerCmd = allocateWalkerSpace(*commandStream, kernel);
 
         if (setupTimestampPacket) {
-            GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(commandStream, pWalkerCmd, currentTimestampPacket,
+            GpgpuWalkerHelper<GfxFamily>::setupTimestampPacket(commandStream, walkerCmd, currentTimestampPacket,
                                                                TimestampPacket::WriteOperationType::AfterWalker);
         }
 
-        auto idd = obtainInterfaceDescriptorData(pWalkerCmd);
+        auto idd = obtainInterfaceDescriptorData(walkerCmd);
 
-        bool localIdsGeneration = KernelCommandsHelper<GfxFamily>::isDispatchForLocalIdsGeneration(dim, globalWorkSizes, localWorkSizes);
-        auto offsetCrossThreadData = KernelCommandsHelper<GfxFamily>::sendIndirectState(
+        bool localIdsGeneration = KernelCommandsHelper<GfxFamily>::isRuntimeLocalIdsGenerationRequired(dim, globalWorkSizes, localWorkSizes);
+        KernelCommandsHelper<GfxFamily>::sendIndirectState(
             *commandStream,
             *dsh,
             *ioh,
@@ -210,35 +209,15 @@ void HardwareInterface<GfxFamily>::dispatchWalker(
             offsetInterfaceDescriptorTable,
             interfaceDescriptorIndex,
             preemptionMode,
+            walkerCmd,
             idd,
             localIdsGeneration);
 
         size_t globalOffsets[3] = {offset.x, offset.y, offset.z};
         size_t startWorkGroups[3] = {swgs.x, swgs.y, swgs.z};
         size_t numWorkGroups[3] = {nwgs.x, nwgs.y, nwgs.z};
-        auto localWorkSize = GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(pWalkerCmd, globalOffsets, startWorkGroups,
-                                                                                    numWorkGroups, localWorkSizes, simd, dim, localIdsGeneration);
-
-        DEBUG_BREAK_IF(offsetCrossThreadData % 64 != 0);
-        setOffsetCrossThreadData(pWalkerCmd, offsetCrossThreadData, interfaceDescriptorIndex);
-        auto sizeCrossThreadData = kernel.getCrossThreadDataSize();
-
-        size_t sizePerThreadDataTotal = 0;
-        if (localIdsGeneration) {
-            auto threadPayload = kernel.getKernelInfo().patchInfo.threadPayload;
-            DEBUG_BREAK_IF(nullptr == threadPayload);
-
-            auto numChannels = PerThreadDataHelper::getNumLocalIdChannels(*threadPayload);
-            auto localIdSizePerThread = PerThreadDataHelper::getLocalIdSizePerThread(simd, numChannels);
-            localIdSizePerThread = std::max(localIdSizePerThread, sizeof(GRF));
-
-            sizePerThreadDataTotal = getThreadsPerWG(simd, localWorkSize) * localIdSizePerThread;
-            DEBUG_BREAK_IF(sizePerThreadDataTotal == 0); // Hardware requires at least 1 GRF of perThreadData for each thread in thread group
-        }
-
-        auto indirectDataLength = alignUp(static_cast<uint32_t>(sizeCrossThreadData + sizePerThreadDataTotal),
-                                          WALKER_TYPE<GfxFamily>::INDIRECTDATASTARTADDRESS_ALIGN_SIZE);
-        pWalkerCmd->setIndirectDataLength(indirectDataLength);
+        GpgpuWalkerHelper<GfxFamily>::setGpgpuWalkerThreadData(walkerCmd, globalOffsets, startWorkGroups,
+                                                               numWorkGroups, localWorkSizes, simd, dim, localIdsGeneration);
 
         dispatchWorkarounds(commandStream, commandQueue, kernel, false);
         currentDispatchIndex++;

runtime/command_queue/hardware_interface_base.inl

@@ -33,22 +33,10 @@ inline void HardwareInterface<GfxFamily>::getDefaultDshSpace(
 template <typename GfxFamily>
 inline typename HardwareInterface<GfxFamily>::INTERFACE_DESCRIPTOR_DATA *
 HardwareInterface<GfxFamily>::obtainInterfaceDescriptorData(
-    WALKER_HANDLE pCmdData) {
-
+    WALKER_TYPE<GfxFamily> *walkerCmd) {
     return nullptr;
 }
 
-template <typename GfxFamily>
-inline void HardwareInterface<GfxFamily>::setOffsetCrossThreadData(
-    WALKER_HANDLE pCmdData,
-    size_t &offsetCrossThreadData,
-    uint32_t &interfaceDescriptorIndex) {
-
-    WALKER_TYPE<GfxFamily> *pCmd = static_cast<WALKER_TYPE<GfxFamily> *>(pCmdData);
-    pCmd->setIndirectDataStartAddress(static_cast<uint32_t>(offsetCrossThreadData));
-    pCmd->setInterfaceDescriptorOffset(interfaceDescriptorIndex++);
-}
-
 template <typename GfxFamily>
 inline void HardwareInterface<GfxFamily>::dispatchWorkarounds(
     LinearStream *commandStream,
@@ -103,4 +91,12 @@ inline void HardwareInterface<GfxFamily>::dispatchProfilingPerfEndCommands(
     }
 }
 
+template <typename GfxFamily>
+inline WALKER_TYPE<GfxFamily> *HardwareInterface<GfxFamily>::allocateWalkerSpace(LinearStream &commandStream,
+                                                                                 const Kernel &kernel) {
+    auto walkerCmd = static_cast<WALKER_TYPE<GfxFamily> *>(commandStream.getSpace(sizeof(WALKER_TYPE<GfxFamily>)));
+    *walkerCmd = GfxFamily::cmdInitGpgpuWalker;
+    return walkerCmd;
+}
+
 } // namespace OCLRT

runtime/helpers/basic_math.h

@@ -1,23 +1,8 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (C) 2017-2018 Intel Corporation
  *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
+ * SPDX-License-Identifier: MIT
  *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #pragma once
@@ -164,5 +149,15 @@ inline size_t computeTotalElementsCount(const Vec3<size_t> &inputVector) {
     return xDim * yDim * zDim;
 }
 
+template <typename T>
+bool isPow2(T val) {
+    if (val != 0) {
+        if ((val & (val - 1)) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 } // namespace Math
 } // namespace OCLRT

runtime/helpers/kernel_commands.h

@@ -24,6 +24,9 @@ class IndirectHeap;
 struct CrossThreadInfo;
 struct MultiDispatchInfo;
 
+template <typename GfxFamily>
+using WALKER_TYPE = typename GfxFamily::WALKER_TYPE;
+
 template <typename GfxFamily>
 struct KernelCommandsHelper : public PerThreadDataHelper {
     using BINDING_TABLE_STATE = typename GfxFamily::BINDING_TABLE_STATE;
@@ -88,8 +91,9 @@ struct KernelCommandsHelper : public PerThreadDataHelper {
         uint32_t simd,
         const size_t localWorkSize[3],
         const uint64_t offsetInterfaceDescriptorTable,
-        const uint32_t interfaceDescriptorIndex,
+        uint32_t &interfaceDescriptorIndex,
         PreemptionMode preemptionMode,
+        WALKER_TYPE<GfxFamily> *walkerCmd,
         INTERFACE_DESCRIPTOR_DATA *inlineInterfaceDescriptor,
         bool localIdsGeneration);
 
@@ -155,6 +159,6 @@ struct KernelCommandsHelper : public PerThreadDataHelper {
 
     static bool doBindingTablePrefetch();
 
-    static bool isDispatchForLocalIdsGeneration(uint32_t workDim, size_t *gws, size_t *lws);
+    static bool isRuntimeLocalIdsGenerationRequired(uint32_t workDim, size_t *gws, size_t *lws);
 };
 } // namespace OCLRT