Kmd notify improvements [1/n]: Quick KMD sleep optimization

- KmdNotifyProperties struct for CapabilityTable that can be extended by
  incoming KmdNotify related optimizations
- Quick KMD sleep optimization that is called from async events handler
- Optimization makes a taskCount check in busy loop with much smaller
  delay than basic version of KMD Notify optimization

Change-Id: I60c851c59895f0cf9de1e1f21e755a8b4c2fe900

Author: BartoszDunajski

runtime/command_queue/command_queue.cpp

@@ -151,13 +151,13 @@ bool CommandQueue::isCompleted(uint32_t taskCount) const {
     return tag >= taskCount;
 }
 
-void CommandQueue::waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushStampToWait) {
+void CommandQueue::waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep) {
     WAIT_ENTER()
 
     DBG_LOG(LogTaskCounts, __FUNCTION__, "Waiting for taskCount:", taskCountToWait);
     DBG_LOG(LogTaskCounts, __FUNCTION__, "Line: ", __LINE__, "Current taskCount:", getHwTag());
 
-    device->getCommandStreamReceiver().waitForTaskCountWithKmdNotifyFallback(taskCountToWait, flushStampToWait);
+    device->getCommandStreamReceiver().waitForTaskCountWithKmdNotifyFallback(taskCountToWait, flushStampToWait, useQuickKmdSleep);
 
     DEBUG_BREAK_IF(getHwTag() < taskCountToWait);
     latestTaskCountWaited = taskCountToWait;

runtime/command_queue/command_queue.h

@@ -318,7 +318,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
 
     MOCKABLE_VIRTUAL bool isQueueBlocked();
 
-    void waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushStampToWait);
+    MOCKABLE_VIRTUAL void waitUntilComplete(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep);
 
     void flushWaitList(cl_uint numEventsInWaitList,
                        const cl_event *eventWaitList,

runtime/command_queue/enqueue_common.h

@@ -325,7 +325,7 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
                 commandStreamReceiver.overrideMediaVFEStateDirty(true);
 
                 if (devQueueHw->getSchedulerReturnInstance() > 0) {
-                    waitUntilComplete(completionStamp.taskCount, completionStamp.flushStamp);
+                    waitUntilComplete(completionStamp.taskCount, completionStamp.flushStamp, false);
 
                     BuiltinKernelsSimulation::SchedulerSimulation<GfxFamily> simulation;
                     simulation.runSchedulerSimulation(devQueueHw->getQueueBuffer(),
@@ -404,9 +404,9 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
         if (blockQueue) {
             while (isQueueBlocked())
                 ;
-            waitUntilComplete(taskCount, flushStamp->peekStamp());
+            waitUntilComplete(taskCount, flushStamp->peekStamp(), false);
         } else {
-            waitUntilComplete(taskCount, flushStamp->peekStamp());
+            waitUntilComplete(taskCount, flushStamp->peekStamp(), false);
             for (auto sIt = surfacesForResidency, sE = surfacesForResidency + numSurfaceForResidency;
                  sIt != sE; ++sIt) {
                 (*sIt)->setCompletionStamp(completionStamp, nullptr, nullptr);

runtime/command_queue/finish.h

@@ -41,7 +41,7 @@ cl_int CommandQueueHw<GfxFamily>::finish(bool dcFlush) {
     auto flushStampToWaitFor = this->flushStamp->peekStamp();
 
     // Stall until HW reaches CQ taskCount
-    waitUntilComplete(taskCountToWaitFor, flushStampToWaitFor);
+    waitUntilComplete(taskCountToWaitFor, flushStampToWaitFor, false);
 
     commandStreamReceiver.waitForTaskCountAndCleanAllocationList(taskCountToWaitFor, TEMPORARY_ALLOCATION);
 

runtime/command_stream/command_stream_receiver.h

@@ -111,7 +111,7 @@ class CommandStreamReceiver {
 
     void requestThreadArbitrationPolicy(uint32_t requiredPolicy) { this->requiredThreadArbitrationPolicy = requiredPolicy; }
 
-    virtual void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait) = 0;
+    virtual void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep) = 0;
     MOCKABLE_VIRTUAL bool waitForCompletionWithTimeout(bool enableTimeout, int64_t timeoutMicroseconds, uint32_t taskCountToWait);
 
     // returns size of block that needs to be reserved at the beginning of each instruction heap for CommandStreamReceiver

runtime/command_stream/command_stream_receiver_hw.h

@@ -65,7 +65,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver {
     size_t getCmdSizeForMediaSampler(bool mediaSamplerRequired) const;
     void programCoherency(LinearStream &csr, DispatchFlags &dispatchFlags);
 
-    void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait) override;
+    void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep) override;
     const HardwareInfo &peekHwInfo() const { return hwInfo; }
 
   protected:

runtime/command_stream/command_stream_receiver_hw.inl

@@ -558,14 +558,18 @@ inline void CommandStreamReceiverHw<GfxFamily>::emitNoop(LinearStream &commandSt
 }
 
 template <typename GfxFamily>
-inline void CommandStreamReceiverHw<GfxFamily>::waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait) {
-    auto status = waitForCompletionWithTimeout(this->hwInfo.capabilityTable.enableKmdNotify && flushStampToWait != 0,
-                                               this->hwInfo.capabilityTable.delayKmdNotifyMicroseconds,
-                                               taskCountToWait);
+inline void CommandStreamReceiverHw<GfxFamily>::waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool useQuickKmdSleep) {
+    const auto &kmdNotifyProperties = this->hwInfo.capabilityTable.kmdNotifyProperties;
+
+    const auto &kmdNotifyDelay = useQuickKmdSleep && kmdNotifyProperties.enableQuickKmdSleep ? kmdNotifyProperties.delayQuickKmdSleepMicroseconds
+                                                                                             : kmdNotifyProperties.delayKmdNotifyMicroseconds;
+
+    auto status = waitForCompletionWithTimeout(kmdNotifyProperties.enableKmdNotify && flushStampToWait != 0,
+                                               kmdNotifyDelay, taskCountToWait);
     if (!status) {
         waitForFlushStamp(flushStampToWait);
         //now call blocking wait, this is to ensure that task count is reached
-        waitForCompletionWithTimeout(false, this->hwInfo.capabilityTable.delayKmdNotifyMicroseconds, taskCountToWait);
+        waitForCompletionWithTimeout(false, kmdNotifyDelay, taskCountToWait);
     }
 
     UNRECOVERABLE_IF(*getTagAddress() < taskCountToWait);

runtime/event/async_events_handler.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -87,7 +87,7 @@ void AsyncEventsHandler::asyncProcess() {
 
         sleepCandidate = processList();
         if (sleepCandidate) {
-            sleepCandidate->wait(true);
+            sleepCandidate->wait(true, true);
         }
         std::this_thread::yield();
     }

runtime/event/event.cpp

@@ -288,14 +288,14 @@ bool Event::calcProfilingData() {
     return dataCalculated;
 }
 
-inline bool Event::wait(bool blocking) {
+inline bool Event::wait(bool blocking, bool useQuickKmdSleep) {
     while (this->taskCount == Event::eventNotReady) {
         if (blocking == false) {
             return false;
         }
     }
 
-    cmdQueue->waitUntilComplete(taskCount.load(), flushStamp->peekStamp());
+    cmdQueue->waitUntilComplete(taskCount.load(), flushStamp->peekStamp(), useQuickKmdSleep);
     updateExecutionStatus();
 
     DEBUG_BREAK_IF(this->taskLevel == Event::eventNotReady && this->executionStatus >= 0);
@@ -495,7 +495,7 @@ cl_int Event::waitForEvents(cl_uint numEvents,
                 return CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST;
             }
 
-            if (event->wait(false) == false) {
+            if (event->wait(false, false) == false) {
                 pendingEventsLeft->push_back(event);
             }
         }

runtime/event/event.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -225,7 +225,7 @@ class Event : public BaseObject<_cl_event>, public IDNode<Event> {
 
     //returns true on success
     //if(blocking==false), will return with false instead of blocking while waiting for completion
-    virtual bool wait(bool blocking);
+    virtual bool wait(bool blocking, bool useQuickKmdSleep);
 
     bool isUserEvent() const {
         return (CL_COMMAND_USER == cmdType);