User space AUBs capable of memory re-dumps on CPU-side memory modifications.

Any CPU related updates such as clEnqueueMapBuffer or similar
need to trigger a re-dump of memory prior to the next clEnqueue call.

Change-Id: I7b31e559278e92ff55b6ebab8ef4190caef1ebc0

Author: smilczar

runtime/command_queue/cpu_data_transfer_handler.cpp

@@ -135,6 +135,10 @@ void *CommandQueue::cpuDataTransferHandler(TransferProperties &transferPropertie
                 }
                 eventCompleted = true;
             }
+            if (!unmapInfo.readOnly) {
+                auto graphicsAllocation = transferProperties.memObj->getGraphicsAllocation();
+                graphicsAllocation->clearTypeAubNonWritable();
+            }
             break;
         case CL_COMMAND_READ_BUFFER:
             memcpy_s(transferProperties.ptr, transferProperties.size[0], ptrOffset(transferProperties.memObj->getCpuAddressForMemoryTransfer(), transferProperties.offset[0]), transferProperties.size[0]);

runtime/command_stream/aub_command_stream_receiver_hw.inl

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 -2018, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -392,7 +392,7 @@ void *AUBCommandStreamReceiverHw<GfxFamily>::flattenBatchBuffer(BatchBuffer &bat
     void *flatBatchBuffer = nullptr;
 
     if (batchBuffer.chainedBatchBuffer) {
-        batchBuffer.chainedBatchBuffer->setAllocationType(batchBuffer.chainedBatchBuffer->getAllocationType() | GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE);
+        batchBuffer.chainedBatchBuffer->setTypeAubNonWritable();
         auto sizeMainBatchBuffer = batchBuffer.chainedBatchBufferStartOffset - batchBuffer.startOffset;
         auto flatBatchBufferSize = alignUp(sizeMainBatchBuffer + batchBuffer.chainedBatchBuffer->getUnderlyingBufferSize(), MemoryConstants::pageSize);
         flatBatchBuffer = this->getMemoryManager()->alignedMallocWrapper(flatBatchBufferSize, MemoryConstants::pageSize);
@@ -488,7 +488,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxA
     auto size = gfxAllocation.getUnderlyingBufferSize();
     auto allocType = gfxAllocation.getAllocationType();
 
-    if ((size == 0) || !!(allocType & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE))
+    if ((size == 0) || gfxAllocation.isTypeAubNonWritable())
         return false;
 
     {
@@ -515,7 +515,7 @@ bool AUBCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxA
 
     if (!!(allocType & GraphicsAllocation::ALLOCATION_TYPE_BUFFER) ||
         !!(allocType & GraphicsAllocation::ALLOCATION_TYPE_IMAGE))
-        gfxAllocation.setAllocationType(allocType | GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE);
+        gfxAllocation.setTypeAubNonWritable();
 
     return true;
 }
@@ -526,8 +526,7 @@ void AUBCommandStreamReceiverHw<GfxFamily>::processResidency(ResidencyContainer
 
     for (auto &gfxAllocation : residencyAllocations) {
         if (!writeMemory(*gfxAllocation)) {
-            DEBUG_BREAK_IF(!((gfxAllocation->getUnderlyingBufferSize() == 0) ||
-                             !!(gfxAllocation->getAllocationType() & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE)));
+            DEBUG_BREAK_IF(!((gfxAllocation->getUnderlyingBufferSize() == 0) || gfxAllocation->isTypeAubNonWritable()));
         }
         gfxAllocation->residencyTaskCount = this->taskCount + 1;
     }

runtime/memory_manager/graphics_allocation.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -112,6 +112,10 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
     void setAllocationType(uint32_t allocationType) { this->allocationType = allocationType; }
     uint32_t getAllocationType() const { return allocationType; }
 
+    void setTypeAubNonWritable() { this->allocationType |= GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE; }
+    void clearTypeAubNonWritable() { this->allocationType &= ~GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE; }
+    bool isTypeAubNonWritable() const { return !!(this->allocationType & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE); }
+
     uint32_t taskCount = ObjectNotUsed;
     OsHandleStorage fragmentsStorage;
     bool isL3Capable();

unit_tests/aub_tests/command_queue/CMakeLists.txt

@@ -27,6 +27,7 @@ target_sources(igdrcl_aub_tests PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_fill_buffer_aub_tests.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_fill_image_aub_tests.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_kernel_aub_tests.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_map_buffer_aub_tests.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_map_image_aub_tests.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_read_buffer_aub_tests.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_read_buffer_rect_aub_tests.cpp

unit_tests/aub_tests/command_queue/enqueue_map_buffer_aub_tests.cpp

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "unit_tests/aub_tests/command_queue/command_enqueue_fixture.h"
+#include "unit_tests/mocks/mock_context.h"
+#include "runtime/mem_obj/buffer.h"
+#include "test.h"
+
+using namespace OCLRT;
+
+struct AUBMapBuffer
+    : public CommandEnqueueAUBFixture,
+      public ::testing::Test {
+
+    void SetUp() override {
+        CommandEnqueueAUBFixture::SetUp();
+    }
+
+    void TearDown() override {
+        CommandEnqueueAUBFixture::TearDown();
+    }
+};
+
+HWTEST_F(AUBMapBuffer, MapUpdateUnmapVerify) {
+    MockContext context(&this->pCmdQ->getDevice());
+    auto retVal = CL_INVALID_VALUE;
+    size_t bufferSize = 10;
+
+    std::unique_ptr<Buffer> buffer(Buffer::create(
+        &context,
+        CL_MEM_READ_WRITE,
+        bufferSize,
+        nullptr,
+        retVal));
+    ASSERT_NE(nullptr, buffer);
+
+    uint8_t pattern[] = {0xFF};
+    size_t patternSize = sizeof(pattern);
+
+    retVal = pCmdQ->enqueueFillBuffer(
+        buffer.get(),
+        pattern,
+        patternSize,
+        0,
+        bufferSize,
+        0,
+        nullptr,
+        nullptr);
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    auto mappedPtr = pCmdQ->enqueueMapBuffer(buffer.get(), CL_TRUE, CL_MAP_WRITE | CL_MAP_READ,
+                                             0, bufferSize, 0, nullptr, nullptr, retVal);
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    // write to mapped ptr
+    auto mappedPtrStart = static_cast<uint8_t *>(mappedPtr);
+    for (uint32_t i = 0; i < bufferSize; i++) {
+        *(mappedPtrStart + i) = i;
+    }
+
+    pCmdQ->enqueueUnmapMemObject(buffer.get(), mappedPtr, 0, nullptr, nullptr);
+
+    // verify unmap
+    std::unique_ptr<uint8_t[]> readMemory(new uint8_t[bufferSize]);
+    buffer->forceDisallowCPUCopy = true;
+
+    retVal = pCmdQ->enqueueReadBuffer(buffer.get(), CL_TRUE, 0, bufferSize, readMemory.get(), 0, nullptr, nullptr);
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    for (size_t i = 0; i < bufferSize; i++) {
+        AUBCommandStreamFixture::expectMemory<FamilyType>(&readMemory[i], &i, sizeof(uint8_t));
+    }
+}

unit_tests/command_queue/enqueue_unmap_memobject_tests.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -203,3 +203,22 @@ TEST_F(EnqueueUnmapMemObjTest, UnmapMemObjWaitEvent) {
     clReleaseEvent(waitEvent);
     clReleaseEvent(retEvent);
 }
+
+HWTEST_F(EnqueueUnmapMemObjTest, givenEnqueueUnmapMemObjectWhenNonAubWritableBufferObjectMappedToHostPtrForWritingThenItShouldBeResetToAubWritable) {
+    auto buffer = std::unique_ptr<Buffer>(BufferHelper<>::create());
+    ASSERT_NE(nullptr, buffer);
+    buffer->getGraphicsAllocation()->setTypeAubNonWritable();
+
+    auto mappedForWritingPtr = pCmdQ->enqueueMapBuffer(buffer.get(), CL_TRUE, CL_MAP_WRITE, 0, 8, 0, nullptr, nullptr, retVal);
+    ASSERT_NE(nullptr, mappedForWritingPtr);
+
+    retVal = pCmdQ->enqueueUnmapMemObject(
+        buffer.get(),
+        mappedForWritingPtr,
+        0,
+        nullptr,
+        nullptr);
+    ASSERT_EQ(CL_SUCCESS, retVal);
+
+    EXPECT_FALSE(buffer->getGraphicsAllocation()->isTypeAubNonWritable());
+}

unit_tests/command_stream/aub_command_stream_receiver_tests.cpp

@@ -307,7 +307,7 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverWhenGraphic
 
     auto gfxAllocation = memoryManager->allocateGraphicsMemory(sizeof(uint32_t), sizeof(uint32_t), false, false);
 
-    EXPECT_FALSE(!!(gfxAllocation->getAllocationType() & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE));
+    EXPECT_FALSE(gfxAllocation->isTypeAubNonWritable());
 
     memoryManager->freeGraphicsMemory(gfxAllocation);
 }
@@ -322,7 +322,7 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverWhenProcess
     ResidencyContainer allocationsForResidency = {gfxDefaultAllocation};
     aubCsr->processResidency(&allocationsForResidency);
 
-    EXPECT_FALSE(!!(gfxDefaultAllocation->getAllocationType() & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE));
+    EXPECT_FALSE(gfxDefaultAllocation->isTypeAubNonWritable());
 
     memoryManager->freeGraphicsMemory(gfxDefaultAllocation);
 }
@@ -341,8 +341,8 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverWhenProcess
     ResidencyContainer allocationsForResidency = {gfxBufferAllocation, gfxImageAllocation};
     aubCsr->processResidency(&allocationsForResidency);
 
-    EXPECT_TRUE(!!(gfxBufferAllocation->getAllocationType() & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE));
-    EXPECT_TRUE(!!(gfxImageAllocation->getAllocationType() & GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE));
+    EXPECT_TRUE(gfxBufferAllocation->isTypeAubNonWritable());
+    EXPECT_TRUE(gfxImageAllocation->isTypeAubNonWritable());
 
     memoryManager->freeGraphicsMemory(gfxBufferAllocation);
     memoryManager->freeGraphicsMemory(gfxImageAllocation);
@@ -367,7 +367,7 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverWhenGraphic
 
     auto gfxAllocation = memoryManager->allocateGraphicsMemory(sizeof(uint32_t), sizeof(uint32_t), false, false);
 
-    gfxAllocation->setAllocationType(GraphicsAllocation::ALLOCATION_TYPE_NON_AUB_WRITABLE);
+    gfxAllocation->setTypeAubNonWritable();
     EXPECT_FALSE(aubCsr->writeMemory(*gfxAllocation));
 
     memoryManager->freeGraphicsMemory(gfxAllocation);

unit_tests/memory_manager/memory_manager_tests.cpp

@@ -133,6 +133,16 @@ TEST(GraphicsAllocationTest, GivenGraphicsAllocationWhenLockingThenIsLocked) {
     EXPECT_NE(0ULL, gpuAddr);
 }
 
+TEST(GraphicsAllocationTest, givenGraphicsAllocationWhenChangingTypeAubNonWritableThenItIsSetCorrectly) {
+    GraphicsAllocation gfxAllocation((void *)0x30000, 0x1000);
+
+    gfxAllocation.setTypeAubNonWritable();
+    EXPECT_TRUE(gfxAllocation.isTypeAubNonWritable());
+
+    gfxAllocation.clearTypeAubNonWritable();
+    EXPECT_FALSE(gfxAllocation.isTypeAubNonWritable());
+}
+
 TEST_F(MemoryAllocatorTest, allocateSystem) {
     auto ptr = memoryManager->allocateSystemMemory(sizeof(char), 0);
     EXPECT_NE(nullptr, ptr);