[L0][XE_HPC]Perform memcpy on CPU for non-usm ptrs

Related-To: NEO-7237

If size is small enough, it is more efficient to
perform copy through locked ptr on CPU.
This change also introduces experimental flag to
enable this.

Signed-off-by: Szymon Morek <szymon.morek@intel.com>

Author: smorek-intel

level_zero/core/source/cmdlist/cmdlist.h

@@ -323,6 +323,9 @@ struct CommandList : _ze_command_list_handle_t {
     bool multiReturnPointCommandList = false;
     bool systolicModeSupport = false;
     bool pipelineSelectStateTracking = false;
+
+    std::atomic<uint32_t> barrierCounter{0u};
+    uint32_t latestFlushedBarrierCounter = 0u;
 };
 
 using CommandListAllocatorFn = CommandList *(*)(uint32_t);

level_zero/core/source/cmdlist/cmdlist_hw.inl

@@ -2492,7 +2492,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendBarrier(ze_event_handle_
     }
 
     appendSignalEventPostWalker(signalEvent, workloadPartition);
-
+    this->barrierCounter++;
     return ZE_RESULT_SUCCESS;
 }
 

level_zero/core/source/cmdlist/cmdlist_hw_immediate.h

@@ -9,6 +9,10 @@
 
 #include "level_zero/core/source/cmdlist/cmdlist_hw.h"
 
+namespace NEO {
+struct SvmAllocationData;
+}
+
 namespace L0 {
 
 struct EventPool;
@@ -123,6 +127,11 @@ struct CommandListCoreFamilyImmediate : public CommandListCoreFamily<gfxCoreFami
 
     void createLogicalStateHelper() override {}
     NEO::LogicalStateHelper *getLogicalStateHelper() const override;
+
+    bool preferCopyThroughLockedPtr(NEO::SvmAllocationData *dstAlloc, bool dstFound, NEO::SvmAllocationData *srcAlloc, bool srcFound, size_t size);
+    bool isAllocUSMDeviceMemory(NEO::SvmAllocationData *alloc, bool allocFound);
+    ze_result_t performCpuMemcpy(void *dstptr, const void *srcptr, size_t size, bool isDstDeviceMemory, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents);
+    void *obtainLockedPtrFromDevice(void *ptr, size_t size);
 };
 
 template <PRODUCT_FAMILY gfxProductFamily>

level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl

@@ -13,6 +13,7 @@
 #include "shared/source/helpers/logical_state_helper.h"
 #include "shared/source/memory_manager/internal_allocation_storage.h"
 #include "shared/source/memory_manager/prefetch_manager.h"
+#include "shared/source/memory_manager/unified_memory_manager.h"
 
 #include "level_zero/core/source/cmdlist/cmdlist_hw_immediate.h"
 #include "level_zero/core/source/device/bcs_split.h"
@@ -227,6 +228,14 @@ ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::appendMemoryCopy(
 
     ze_result_t ret;
 
+    NEO::SvmAllocationData *srcAllocData = nullptr;
+    NEO::SvmAllocationData *dstAllocData = nullptr;
+    bool srcAllocFound = this->device->getDriverHandle()->findAllocationDataForRange(const_cast<void *>(srcptr), size, &srcAllocData);
+    bool dstAllocFound = this->device->getDriverHandle()->findAllocationDataForRange(dstptr, size, &dstAllocData);
+    if (preferCopyThroughLockedPtr(dstAllocData, dstAllocFound, srcAllocData, srcAllocFound, size)) {
+        return performCpuMemcpy(dstptr, srcptr, size, dstAllocFound, hSignalEvent, numWaitEvents, phWaitEvents);
+    }
+
     if (this->isAppendSplitNeeded(dstptr, srcptr, size)) {
         ret = static_cast<DeviceImp *>(this->device)->bcsSplit.appendSplitCall<gfxCoreFamily, void *, const void *>(this, dstptr, srcptr, size, hSignalEvent, [&](void *dstptrParam, const void *srcptrParam, size_t sizeParam, ze_event_handle_t hSignalEventParam) {
             return CommandListCoreFamily<gfxCoreFamily>::appendMemoryCopy(dstptrParam, srcptrParam, sizeParam, hSignalEventParam, numWaitEvents, phWaitEvents);
@@ -461,4 +470,91 @@ ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::flushImmediate(ze_res
     return inputRet;
 }
 
+template <GFXCORE_FAMILY gfxCoreFamily>
+bool CommandListCoreFamilyImmediate<gfxCoreFamily>::preferCopyThroughLockedPtr(NEO::SvmAllocationData *dstAlloc, bool dstFound, NEO::SvmAllocationData *srcAlloc, bool srcFound, size_t size) {
+    size_t h2DThreshold = 2 * MemoryConstants::megaByte;
+    size_t d2HThreshold = 1 * MemoryConstants::kiloByte;
+    if (NEO::DebugManager.flags.ExperimentalH2DCpuCopyThreshold.get() != -1) {
+        h2DThreshold = NEO::DebugManager.flags.ExperimentalH2DCpuCopyThreshold.get();
+    }
+    if (NEO::DebugManager.flags.ExperimentalD2HCpuCopyThreshold.get() != -1) {
+        d2HThreshold = NEO::DebugManager.flags.ExperimentalD2HCpuCopyThreshold.get();
+    }
+    if (NEO::HwHelper::get(this->device->getHwInfo().platform.eRenderCoreFamily).copyThroughLockedPtrEnabled()) {
+        return (!srcFound && isAllocUSMDeviceMemory(dstAlloc, dstFound) && size <= h2DThreshold) ||
+               (!dstFound && isAllocUSMDeviceMemory(srcAlloc, srcFound) && size <= d2HThreshold);
+    }
+    return false;
+}
+
+template <GFXCORE_FAMILY gfxCoreFamily>
+bool CommandListCoreFamilyImmediate<gfxCoreFamily>::isAllocUSMDeviceMemory(NEO::SvmAllocationData *alloc, bool allocFound) {
+    return allocFound && (alloc->memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY);
+}
+
+template <GFXCORE_FAMILY gfxCoreFamily>
+ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::performCpuMemcpy(void *dstptr, const void *srcptr, size_t size, bool isDstDeviceMemory, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) {
+
+    bool needsBarrier = (numWaitEvents > 0);
+    if (needsBarrier) {
+        this->appendBarrier(nullptr, numWaitEvents, phWaitEvents);
+    }
+
+    bool needsFlushTagUpdate = this->latestFlushedBarrierCounter < this->barrierCounter;
+    if (needsFlushTagUpdate) {
+        this->csr->flushTagUpdate();
+    }
+
+    Event *signalEvent = nullptr;
+    if (hSignalEvent) {
+        signalEvent = Event::fromHandle(hSignalEvent);
+    }
+
+    const void *cpuMemcpySrcPtr = nullptr;
+    void *cpuMemcpyDstPtr = nullptr;
+    if (isDstDeviceMemory) {
+        cpuMemcpySrcPtr = srcptr;
+        cpuMemcpyDstPtr = obtainLockedPtrFromDevice(dstptr, size);
+    } else {
+        cpuMemcpySrcPtr = obtainLockedPtrFromDevice(const_cast<void *>(srcptr), size);
+        cpuMemcpyDstPtr = dstptr;
+    }
+
+    if (needsFlushTagUpdate) {
+        auto timeoutMicroseconds = NEO::TimeoutControls::maxTimeout;
+        const auto waitStatus = this->csr->waitForCompletionWithTimeout(NEO::WaitParams{false, false, timeoutMicroseconds}, this->csr->peekTaskCount());
+        if (waitStatus == NEO::WaitStatus::GpuHang) {
+            return ZE_RESULT_ERROR_DEVICE_LOST;
+        }
+        this->latestFlushedBarrierCounter = this->barrierCounter;
+    }
+
+    if (signalEvent) {
+        signalEvent->setGpuStartTimestamp();
+    }
+
+    memcpy_s(cpuMemcpyDstPtr, size, cpuMemcpySrcPtr, size);
+
+    if (signalEvent) {
+        signalEvent->setGpuEndTimestamp();
+        signalEvent->hostSignal();
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+template <GFXCORE_FAMILY gfxCoreFamily>
+void *CommandListCoreFamilyImmediate<gfxCoreFamily>::obtainLockedPtrFromDevice(void *ptr, size_t size) {
+    NEO::SvmAllocationData *allocData = nullptr;
+    auto allocFound = this->device->getDriverHandle()->findAllocationDataForRange(ptr, size, &allocData);
+    UNRECOVERABLE_IF(!allocFound);
+
+    auto alloc = allocData->gpuAllocations.getGraphicsAllocation(this->device->getRootDeviceIndex());
+    if (!alloc->isLocked()) {
+        this->device->getDriverHandle()->getMemoryManager()->lockResource(alloc);
+    }
+    auto gpuAddress = allocData->gpuAllocations.getGraphicsAllocation(this->device->getRootDeviceIndex())->getGpuAddress();
+    auto offset = ptrDiff(ptr, gpuAddress);
+    return ptrOffset(alloc->getLockedPtr(), offset);
+}
+
 } // namespace L0

level_zero/core/source/event/event.h

@@ -65,6 +65,8 @@ struct Event : _ze_event_handle_t {
     void *getHostAddress() { return hostAddress; }
     virtual void setPacketsInUse(uint32_t value) = 0;
     uint32_t getCurrKernelDataIndex() const { return kernelCount - 1; }
+    virtual void setGpuStartTimestamp() = 0;
+    virtual void setGpuEndTimestamp() = 0;
 
     size_t getContextStartOffset() const {
         return contextStartOffset;
@@ -143,6 +145,10 @@ struct Event : _ze_event_handle_t {
     size_t singlePacketSize = 0u;
     size_t eventPoolOffset = 0u;
 
+    size_t cpuStartTimestamp = 0u;
+    size_t gpuStartTimestamp = 0u;
+    size_t gpuEndTimestamp = 0u;
+
     uint32_t kernelCount = 1u;
 
     bool isTimestampEvent = false;
@@ -195,6 +201,8 @@ struct EventImp : public Event {
     uint32_t getPacketsInUse() override;
     uint32_t getPacketsUsedInLastKernel() override;
     void setPacketsInUse(uint32_t value) override;
+    void setGpuStartTimestamp() override;
+    void setGpuEndTimestamp() override;
 
     std::unique_ptr<KernelEventCompletionData<TagSizeT>[]> kernelEventCompletionData;
 

level_zero/core/source/event/event_impl.inl

@@ -7,6 +7,7 @@
 
 #include "shared/source/debug_settings/debug_settings_manager.h"
 #include "shared/source/memory_manager/internal_allocation_storage.h"
+#include "shared/source/os_interface/os_time.h"
 
 #include "level_zero/core/source/event/event.h"
 #include "level_zero/core/source/hw_helpers/l0_hw_helper.h"
@@ -167,18 +168,24 @@ template <typename TagSizeT>
 ze_result_t EventImp<TagSizeT>::hostEventSetValueTimestamps(TagSizeT eventVal) {
 
     auto baseAddr = castToUint64(hostAddress);
-
-    auto eventTsSetFunc = [&eventVal](auto tsAddr) {
+    auto eventTsSetFunc = [](auto tsAddr, TagSizeT value) {
         auto tsptr = reinterpret_cast<void *>(tsAddr);
-        memcpy_s(tsptr, sizeof(TagSizeT), static_cast<void *>(&eventVal), sizeof(TagSizeT));
+        memcpy_s(tsptr, sizeof(TagSizeT), static_cast<void *>(&value), sizeof(TagSizeT));
     };
+
+    TagSizeT timestampStart = eventVal;
+    TagSizeT timestampEnd = eventVal;
+    if (eventVal == Event::STATE_SIGNALED) {
+        timestampStart = static_cast<TagSizeT>(this->gpuStartTimestamp);
+        timestampEnd = static_cast<TagSizeT>(this->gpuEndTimestamp);
+    }
     for (uint32_t i = 0; i < kernelCount; i++) {
         uint32_t packetsToSet = kernelEventCompletionData[i].getPacketsUsed();
         for (uint32_t j = 0; j < packetsToSet; j++) {
-            eventTsSetFunc(baseAddr + contextStartOffset);
-            eventTsSetFunc(baseAddr + globalStartOffset);
-            eventTsSetFunc(baseAddr + contextEndOffset);
-            eventTsSetFunc(baseAddr + globalEndOffset);
+            eventTsSetFunc(baseAddr + contextStartOffset, timestampStart);
+            eventTsSetFunc(baseAddr + globalStartOffset, timestampStart);
+            eventTsSetFunc(baseAddr + contextEndOffset, timestampEnd);
+            eventTsSetFunc(baseAddr + globalEndOffset, timestampEnd);
             baseAddr += singlePacketSize;
         }
     }
@@ -316,7 +323,6 @@ ze_result_t EventImp<TagSizeT>::queryKernelTimestamp(ze_kernel_timestamp_result_
         eventTsSetFunc(globalEndTS, result.context.kernelEnd);
         eventTsSetFunc(globalEndTS, result.global.kernelEnd);
     }
-
     return ZE_RESULT_SUCCESS;
 }
 
@@ -379,6 +385,9 @@ void EventImp<TagSizeT>::resetPackets() {
         kernelEventCompletionData[i].setPacketsUsed(1);
     }
     kernelCount = 1;
+    cpuStartTimestamp = 0;
+    gpuStartTimestamp = 0;
+    gpuEndTimestamp = 0;
 }
 
 template <typename TagSizeT>
@@ -410,4 +419,21 @@ uint64_t EventImp<TagSizeT>::getPacketAddress(Device *device) {
     return address;
 }
 
+template <typename TagSizeT>
+void EventImp<TagSizeT>::setGpuStartTimestamp() {
+    if (isEventTimestampFlagSet()) {
+        this->device->getGlobalTimestamps(&cpuStartTimestamp, &gpuStartTimestamp);
+        cpuStartTimestamp = cpuStartTimestamp / this->device->getNEODevice()->getDeviceInfo().outProfilingTimerResolution;
+    }
+}
+
+template <typename TagSizeT>
+void EventImp<TagSizeT>::setGpuEndTimestamp() {
+    if (isEventTimestampFlagSet()) {
+        auto resolution = this->device->getNEODevice()->getDeviceInfo().outProfilingTimerResolution;
+        auto cpuEndTimestamp = this->device->getNEODevice()->getOSTime()->getCpuRawTimestamp() / resolution;
+        this->gpuEndTimestamp = gpuStartTimestamp + (cpuEndTimestamp - cpuStartTimestamp);
+    }
+}
+
 } // namespace L0

level_zero/core/test/unit_tests/mocks/mock_event.h

@@ -67,8 +67,11 @@ struct Mock<EventPool> : public EventPool {
 
 class MockEvent : public ::L0::Event {
   public:
+    using ::L0::Event::gpuEndTimestamp;
+    using ::L0::Event::gpuStartTimestamp;
     using ::L0::Event::isCompleted;
     using ::L0::Event::l3FlushAppliedOnKernel;
+
     MockEvent() {
         mockAllocation.reset(new NEO::MockGraphicsAllocation(0,
                                                              NEO::AllocationType::INTERNAL_HOST_MEMORY,
@@ -119,7 +122,8 @@ class MockEvent : public ::L0::Event {
     void resetPackets() override {}
     void setPacketsInUse(uint32_t value) override {}
     uint64_t getPacketAddress(L0::Device *) override { return 0; }
-
+    void setGpuStartTimestamp() override {}
+    void setGpuEndTimestamp() override {}
     std::unique_ptr<NEO::GraphicsAllocation> mockAllocation;
 };
 

level_zero/core/test/unit_tests/sources/event/test_event.cpp

@@ -2202,6 +2202,9 @@ HWTEST_F(EventTests,
 }
 
 struct MockEventCompletion : public EventImp<uint32_t> {
+    using EventImp<uint32_t>::gpuStartTimestamp;
+    using EventImp<uint32_t>::gpuEndTimestamp;
+
     MockEventCompletion(L0::EventPool *eventPool, int index, L0::Device *device) : EventImp(eventPool, index, device) {
         auto neoDevice = device->getNEODevice();
         kernelEventCompletionData = std::make_unique<KernelEventCompletionData<uint32_t>[]>(EventPacketsCount::maxKernelSplit);
@@ -2260,5 +2263,14 @@ TEST_F(EventTests, WhenQueryingStatusAfterResetThenAccessMemory) {
     EXPECT_EQ(event->assignKernelEventCompletionDataCounter, 2u);
 }
 
+TEST_F(EventTests, WhenResetEventThenZeroCpuTimestamps) {
+    auto event = std::make_unique<MockEventCompletion>(eventPool, 1u, device);
+    event->gpuStartTimestamp = 10u;
+    event->gpuEndTimestamp = 20u;
+    EXPECT_EQ(event->reset(), ZE_RESULT_SUCCESS);
+    EXPECT_EQ(event->gpuStartTimestamp, 0u);
+    EXPECT_EQ(event->gpuEndTimestamp, 0u);
+}
+
 } // namespace ult
 } // namespace L0