Extend batch buffer flattening in AubCSR to BatchedDispatch mode

    - batch buffer flatening in batched mode
    - added MI_USER_INTERRUPT command
    - added GUC Work Queue Item

Change-Id: I35142da34b30d3006bb4ffc1521db7f6ebe68ebc

Author: pwilma

runtime/command_queue/command_queue_hw.h

@@ -65,7 +65,7 @@ class CommandQueueHw : public CommandQueue {
         }
 
         if (getCmdQueueProperties<cl_queue_properties>(properties, CL_QUEUE_PROPERTIES) & static_cast<cl_queue_properties>(CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE)) {
-            device->getCommandStreamReceiver().overrideDispatchPolicy(CommandStreamReceiver::BatchedDispatch);
+            device->getCommandStreamReceiver().overrideDispatchPolicy(DispatchMode::BatchedDispatch);
         }
     }
 

runtime/command_queue/enqueue_common.h

@@ -259,7 +259,7 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
         if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
             for (auto &dispatchInfo : multiDispatchInfo) {
                 for (auto &patchInfoData : dispatchInfo.getKernel()->getPatchInfoDataList()) {
-                    commandStreamReceiver.setPatchInfoData(patchInfoData);
+                    commandStreamReceiver.getFlatBatchBufferHelper().setPatchInfoData(patchInfoData);
                 }
             }
         }

runtime/command_stream/aub_command_stream_receiver_hw.h

@@ -63,13 +63,10 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverHw<GfxFamily> {
 
     MemoryManager *createMemoryManager(bool enable64kbPages) override {
         this->memoryManager = new OsAgnosticMemoryManager(enable64kbPages);
+        this->flatBatchBufferHelper->setMemoryManager(this->memoryManager);
         return this->memoryManager;
     }
 
-    bool setPatchInfoData(PatchInfoData &data) override;
-
-    std::vector<PatchInfoData> patchInfoCollection;
-
     static const AubMemDump::LrcaHelper &getCsTraits(EngineType engineType);
 
     struct EngineInfo {
@@ -91,7 +88,8 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverHw<GfxFamily> {
     // remap CPU VA -> GGTT VA
     AddressMapper gttRemap;
 
-    MOCKABLE_VIRTUAL void *flattenBatchBuffer(BatchBuffer &batchBuffer, size_t &sizeBatchBuffer);
     MOCKABLE_VIRTUAL bool addPatchInfoComments();
+    void addGUCStartMessage(uint64_t batchBufferAddress, EngineType engineType);
+    uint32_t getGUCWorkQueueItemHeader(EngineType engineType);
 };
 } // namespace OCLRT

runtime/command_stream/aub_command_stream_receiver_hw.inl

@@ -37,9 +37,9 @@ AUBCommandStreamReceiverHw<GfxFamily>::AUBCommandStreamReceiverHw(const Hardware
     : BaseClass(hwInfoIn),
       stream(std::unique_ptr<AUBCommandStreamReceiver::AubFileStream>(new AUBCommandStreamReceiver::AubFileStream())),
       standalone(standalone) {
-    this->dispatchMode = CommandStreamReceiver::DispatchMode::BatchedDispatch;
+    this->dispatchMode = DispatchMode::BatchedDispatch;
     if (DebugManager.flags.CsrDispatchMode.get()) {
-        this->dispatchMode = (CommandStreamReceiver::DispatchMode)DebugManager.flags.CsrDispatchMode.get();
+        this->dispatchMode = (DispatchMode)DebugManager.flags.CsrDispatchMode.get();
     }
     for (auto &engineInfo : engineInfoTable) {
         engineInfo.pLRCA = nullptr;
@@ -221,8 +221,8 @@ FlushStamp AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer
     auto sizeBatchBuffer = currentOffset - batchBuffer.startOffset;
 
     std::unique_ptr<void, std::function<void(void *)>> flatBatchBuffer(nullptr, [&](void *ptr) { this->getMemoryManager()->alignedFreeWrapper(ptr); });
-    if (DebugManager.flags.FlattenBatchBufferForAUBDump.get() && (this->dispatchMode == CommandStreamReceiver::DispatchMode::ImmediateDispatch)) {
-        flatBatchBuffer.reset(flattenBatchBuffer(batchBuffer, sizeBatchBuffer));
+    if (DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
+        flatBatchBuffer.reset(this->flatBatchBufferHelper->flattenBatchBuffer(batchBuffer, sizeBatchBuffer, this->dispatchMode));
         if (flatBatchBuffer.get() != nullptr) {
             pBatchBuffer = flatBatchBuffer.get();
         }
@@ -248,7 +248,7 @@ FlushStamp AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer
     }
 
     if (this->standalone) {
-        if (this->dispatchMode == CommandStreamReceiver::DispatchMode::ImmediateDispatch) {
+        if (this->dispatchMode == DispatchMode::ImmediateDispatch) {
             if (!DebugManager.flags.FlattenBatchBufferForAUBDump.get()) {
                 makeResident(*batchBuffer.commandBufferAllocation);
             }
@@ -259,6 +259,7 @@ FlushStamp AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer
         processResidency(allocationsForResidency);
     }
     if (DebugManager.flags.AddPatchInfoCommentsForAUBDump.get()) {
+        addGUCStartMessage(static_cast<uint64_t>(reinterpret_cast<std::uintptr_t>(pBatchBuffer)), engineType);
         addPatchInfoComments();
     }
 
@@ -387,32 +388,13 @@ FlushStamp AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer
     return 0;
 }
 
-template <typename GfxFamily>
-void *AUBCommandStreamReceiverHw<GfxFamily>::flattenBatchBuffer(BatchBuffer &batchBuffer, size_t &sizeBatchBuffer) {
-    void *flatBatchBuffer = nullptr;
-
-    if (batchBuffer.chainedBatchBuffer) {
-        batchBuffer.chainedBatchBuffer->setTypeAubNonWritable();
-        auto sizeMainBatchBuffer = batchBuffer.chainedBatchBufferStartOffset - batchBuffer.startOffset;
-        auto flatBatchBufferSize = alignUp(sizeMainBatchBuffer + batchBuffer.chainedBatchBuffer->getUnderlyingBufferSize(), MemoryConstants::pageSize);
-        flatBatchBuffer = this->getMemoryManager()->alignedMallocWrapper(flatBatchBufferSize, MemoryConstants::pageSize);
-        UNRECOVERABLE_IF(flatBatchBuffer == nullptr);
-        // Copy FLB
-        memcpy_s(flatBatchBuffer, sizeMainBatchBuffer, ptrOffset(batchBuffer.commandBufferAllocation->getUnderlyingBuffer(), batchBuffer.startOffset), sizeMainBatchBuffer);
-        // Copy SLB
-        memcpy_s(ptrOffset(flatBatchBuffer, sizeMainBatchBuffer), batchBuffer.chainedBatchBuffer->getUnderlyingBufferSize(), batchBuffer.chainedBatchBuffer->getUnderlyingBuffer(), batchBuffer.chainedBatchBuffer->getUnderlyingBufferSize());
-        sizeBatchBuffer = flatBatchBufferSize;
-    }
-    return flatBatchBuffer;
-}
-
 template <typename GfxFamily>
 bool AUBCommandStreamReceiverHw<GfxFamily>::addPatchInfoComments() {
     std::map<uint64_t, uint64_t> allocationsMap;
 
     std::ostringstream str;
     str << "PatchInfoData" << std::endl;
-    for (auto &patchInfoData : this->patchInfoCollection) {
+    for (auto &patchInfoData : this->flatBatchBufferHelper->getPatchInfoCollection()) {
         str << std::hex << patchInfoData.sourceAllocation << ";";
         str << std::hex << patchInfoData.sourceAllocationOffset << ";";
         str << std::hex << patchInfoData.sourceType << ";";
@@ -432,7 +414,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::addPatchInfoComments() {
         }
     }
     bool result = stream->addComment(str.str().c_str());
-    this->patchInfoCollection.clear();
+    this->flatBatchBufferHelper->getPatchInfoCollection().clear();
     if (!result) {
         return false;
     }
@@ -547,8 +529,41 @@ void AUBCommandStreamReceiverHw<GfxFamily>::addContextToken() {
 }
 
 template <typename GfxFamily>
-bool AUBCommandStreamReceiverHw<GfxFamily>::setPatchInfoData(PatchInfoData &data) {
-    patchInfoCollection.push_back(data);
-    return true;
+void AUBCommandStreamReceiverHw<GfxFamily>::addGUCStartMessage(uint64_t batchBufferAddress, EngineType engineType) {
+    typedef typename GfxFamily::MI_BATCH_BUFFER_START MI_BATCH_BUFFER_START;
+
+    auto bufferSize = sizeof(uint32_t) + sizeof(MI_BATCH_BUFFER_START);
+
+    std::unique_ptr<void, std::function<void(void *)>> buffer(this->getMemoryManager()->alignedMallocWrapper(bufferSize, MemoryConstants::pageSize), [&](void *ptr) { this->getMemoryManager()->alignedFreeWrapper(ptr); });
+    LinearStream linearStream(buffer.get(), bufferSize);
+
+    uint32_t *header = static_cast<uint32_t *>(linearStream.getSpace(sizeof(uint32_t)));
+    *header = getGUCWorkQueueItemHeader(engineType);
+
+    MI_BATCH_BUFFER_START *miBatchBufferStart = linearStream.getSpaceForCmd<MI_BATCH_BUFFER_START>();
+    DEBUG_BREAK_IF(bufferSize != linearStream.getUsed());
+    miBatchBufferStart->init();
+    miBatchBufferStart->setBatchBufferStartAddressGraphicsaddress472(AUB::ptrToPPGTT(buffer.get()));
+    miBatchBufferStart->setAddressSpaceIndicator(MI_BATCH_BUFFER_START::ADDRESS_SPACE_INDICATOR_PPGTT);
+
+    auto physBufferAddres = ppgtt.map(reinterpret_cast<uintptr_t>(buffer.get()), bufferSize);
+    AUB::reserveAddressPPGTT(*stream, reinterpret_cast<uintptr_t>(buffer.get()), bufferSize, physBufferAddres);
+
+    AUB::addMemoryWrite(
+        *stream,
+        physBufferAddres,
+        buffer.get(),
+        bufferSize,
+        AubMemDump::AddressSpaceValues::TraceNonlocal);
+
+    PatchInfoData patchInfoData(batchBufferAddress, 0u, PatchInfoAllocationType::Default, reinterpret_cast<uintptr_t>(buffer.get()), sizeof(uint32_t) + sizeof(MI_BATCH_BUFFER_START) - sizeof(uint64_t), PatchInfoAllocationType::GUCStartMessage);
+    this->flatBatchBufferHelper->setPatchInfoData(patchInfoData);
 }
+
+template <typename GfxFamily>
+uint32_t AUBCommandStreamReceiverHw<GfxFamily>::getGUCWorkQueueItemHeader(EngineType engineType) {
+    uint32_t GUCWorkQueueItemHeader = 0x00030001;
+    return GUCWorkQueueItemHeader;
+}
+
 } // namespace OCLRT

runtime/command_stream/command_stream_receiver.cpp

@@ -127,6 +127,9 @@ MemoryManager *CommandStreamReceiver::getMemoryManager() {
 
 void CommandStreamReceiver::setMemoryManager(MemoryManager *mm) {
     memoryManager = mm;
+    if (flatBatchBufferHelper) {
+        flatBatchBufferHelper->setMemoryManager(mm);
+    }
 }
 
 LinearStream &CommandStreamReceiver::getCS(size_t minRequiredSize) {

runtime/command_stream/command_stream_receiver.h

@@ -27,6 +27,7 @@
 #include "runtime/helpers/completion_stamp.h"
 #include "runtime/helpers/aligned_memory.h"
 #include "runtime/helpers/address_patch.h"
+#include "runtime/helpers/flat_batch_buffer_helper.h"
 #include "runtime/command_stream/csr_definitions.h"
 #include <cstddef>
 #include <cstdint>
@@ -40,16 +41,16 @@ class MemoryManager;
 class OSInterface;
 class GraphicsAllocation;
 
+enum class DispatchMode {
+    DeviceDefault = 0,          //default for given device
+    ImmediateDispatch,          //everything is submitted to the HW immediately
+    AdaptiveDispatch,           //dispatching is handled to async thread, which combines batch buffers basing on load (not implemented)
+    BatchedDispatchWithCounter, //dispatching is batched, after n commands there is implicit flush (not implemented)
+    BatchedDispatch             // dispatching is batched, explicit clFlush is required
+};
+
 class CommandStreamReceiver {
   public:
-    enum DispatchMode {
-        DeviceDefault = 0,          //default for given device
-        ImmediateDispatch,          //everything is submitted to the HW immediately
-        AdaptiveDispatch,           //dispatching is handled to async thread, which combines batch buffers basing on load (not implemented)
-        BatchedDispatchWithCounter, //dispatching is batched, after n commands there is implicit flush (not implemented)
-        BatchedDispatch             // dispatching is batched, explicit clFlush is required
-    };
-
     enum class SamplerCacheFlushState {
         samplerCacheFlushNotRequired,
         samplerCacheFlushBefore, //add sampler cache flush before Walker with redescribed image
@@ -102,7 +103,7 @@ class CommandStreamReceiver {
 
     uint32_t peekLatestFlushedTaskCount() const { return latestFlushedTaskCount; }
 
-    void overrideDispatchPolicy(CommandStreamReceiver::DispatchMode overrideValue) { this->dispatchMode = overrideValue; }
+    void overrideDispatchPolicy(DispatchMode overrideValue) { this->dispatchMode = overrideValue; }
 
     virtual void overrideMediaVFEStateDirty(bool dirty) { mediaVfeStateDirty = dirty; }
 
@@ -122,8 +123,8 @@ class CommandStreamReceiver {
 
     void setSamplerCacheFlushRequired(SamplerCacheFlushState value) { this->samplerCacheFlushRequired = value; }
 
-    // Collect patch info data
-    virtual bool setPatchInfoData(PatchInfoData &data) { return false; }
+    FlatBatchBufferHelper &getFlatBatchBufferHelper() { return *flatBatchBufferHelper.get(); }
+    void overwriteFlatBatchBufferHelper(FlatBatchBufferHelper *newHelper) { flatBatchBufferHelper.reset(newHelper); }
 
   protected:
     void setDisableL3Cache(bool val) {
@@ -167,11 +168,12 @@ class CommandStreamReceiver {
     std::unique_ptr<OSInterface> osInterface;
     std::unique_ptr<SubmissionAggregator> submissionAggregator;
 
-    DispatchMode dispatchMode = ImmediateDispatch;
+    DispatchMode dispatchMode = DispatchMode::ImmediateDispatch;
     bool disableL3Cache = false;
     uint32_t requiredScratchSize = 0;
     uint64_t totalMemoryUsed = 0u;
     SamplerCacheFlushState samplerCacheFlushRequired = SamplerCacheFlushState::samplerCacheFlushNotRequired;
+    std::unique_ptr<FlatBatchBufferHelper> flatBatchBufferHelper;
 };
 
 typedef CommandStreamReceiver *(*CommandStreamReceiverCreateFunc)(const HardwareInfo &hwInfoIn, bool withAubDump);

runtime/command_stream/command_stream_receiver_hw.h

@@ -55,7 +55,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     int getRequiredPipeControlSize() const;
 
     static void addBatchBufferEnd(LinearStream &commandStream, void **patchLocation);
-    static void addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress);
+    void addBatchBufferStart(MI_BATCH_BUFFER_START *commandBufferMemory, uint64_t startAddress);
     static void alignToCacheLine(LinearStream &commandStream);
 
     size_t getRequiredCmdStreamSize(const DispatchFlags &dispatchFlags);
@@ -71,13 +71,12 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     const HardwareInfo &peekHwInfo() const { return hwInfo; }
 
     void collectStateBaseAddresPatchInfo(
-        uint64_t baseAddress,
+        uint64_t commandBufferAddress,
         uint64_t commandOffset,
         const LinearStream &dsh,
         const LinearStream &ioh,
         const LinearStream &ssh,
-        uint64_t generalStateBase,
-        uint64_t internalHeapBaseAddress);
+        uint64_t generalStateBase);
 
     void resetKmdNotifyHelper(KmdNotifyHelper *newHelper);