Skip to content

Commit 1ed273d

Browse files
raiyanlaCompute-Runtime-Automation
raiyanla
authored and
Compute-Runtime-Automation
committed
Ensure shared allocations made resident across multiple devices
Signed-off-by: Raiyan Latif <raiyan.latif@intel.com>
1 parent 8a6d6dd commit 1ed273d

File tree

6 files changed

+95
-6
lines changed

6 files changed

+95
-6
lines changed

‎level_zero/core/source/cmdlist/cmdlist_hw.inl‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,10 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemoryFill(void *ptr,
14961496
NEO::SvmAllocationData *allocData = nullptr;
14971497
bool dstAllocFound = device->getDriverHandle()->findAllocationDataForRange(ptr, size, &allocData);
14981498
if (dstAllocFound) {
1499+
if (allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
1500+
DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(device->getDriverHandle());
1501+
driverHandleImp->makeMemoryResident(device->toHandle(), ptr, size);
1502+
}
14991503
if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY ||
15001504
allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
15011505
hostPointerNeedsFlush = true;
@@ -1843,6 +1847,11 @@ inline AlignedAllocationData CommandListCoreFamily<gfxCoreFamily>::getAlignedAll
18431847
alignedPtr = sourcePtr;
18441848
}
18451849

1850+
if (allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
1851+
DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(device->getDriverHandle());
1852+
driverHandleImp->makeMemoryResident(device->toHandle(), ptr, bufferSize);
1853+
}
1854+
18461855
if (allocData->memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY ||
18471856
allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
18481857
hostPointerNeedsFlush = true;

‎level_zero/core/source/driver/driver_handle.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ struct DriverHandle : _ze_driver_handle_t {
6565
size_t size,
6666
uint32_t rootDeviceIndex,
6767
uintptr_t *gpuAddress) = 0;
68+
virtual ze_result_t makeMemoryResident(ze_device_handle_t hDevice,
69+
void *ptr,
70+
size_t size) = 0;
6871

6972
static DriverHandle *fromHandle(ze_driver_handle_t handle) { return static_cast<DriverHandle *>(handle); }
7073
inline ze_driver_handle_t toHandle() { return this; }

‎level_zero/core/source/driver/driver_handle_imp.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,28 @@ NEO::GraphicsAllocation *DriverHandleImp::getPeerAllocation(Device *device,
578578
return alloc;
579579
}
580580

581+
ze_result_t DriverHandleImp::makeMemoryResident(ze_device_handle_t hDevice, void *ptr, size_t size) {
582+
Device *device = L0::Device::fromHandle(hDevice);
583+
NEO::Device *neoDevice = device->getNEODevice();
584+
auto allocation = device->getDriverHandle()->getDriverSystemMemoryAllocation(
585+
ptr,
586+
size,
587+
neoDevice->getRootDeviceIndex(),
588+
nullptr);
589+
if (allocation == nullptr) {
590+
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
591+
}
592+
593+
auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
594+
if (allocData && allocData->memoryType == InternalMemoryType::SHARED_UNIFIED_MEMORY) {
595+
DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(device->getDriverHandle());
596+
std::lock_guard<std::mutex> lock(driverHandleImp->sharedMakeResidentAllocationsLock);
597+
driverHandleImp->sharedMakeResidentAllocations.insert({ptr, allocation});
598+
}
599+
600+
return ZE_RESULT_SUCCESS;
601+
}
602+
581603
void *DriverHandleImp::importNTHandle(ze_device_handle_t hDevice, void *handle) {
582604
auto neoDevice = Device::fromHandle(hDevice)->getNEODevice();
583605

‎level_zero/core/source/driver/driver_handle_imp.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ struct DriverHandleImp : public DriverHandle {
8383
std::mutex sharedMakeResidentAllocationsLock;
8484
std::map<void *, NEO::GraphicsAllocation *> sharedMakeResidentAllocations;
8585

86+
ze_result_t makeMemoryResident(ze_device_handle_t hDevice,
87+
void *ptr,
88+
size_t size) override;
89+
8690
std::vector<Device *> devices;
8791
// Spec extensions
8892
const std::vector<std::pair<std::string, uint32_t>> extensionsSupported = {

‎level_zero/core/test/unit_tests/sources/context/test_context.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -459,14 +459,14 @@ struct ContextMakeMemoryResidentAndMigrationTests : public ContextMakeMemoryResi
459459
struct MockResidentTestsPageFaultManager : public MockPageFaultManager {
460460
void moveAllocationToGpuDomain(void *ptr) override {
461461
moveAllocationToGpuDomainCalledTimes++;
462-
migratedAddress = ptr;
462+
migratedAddress.push_back(ptr);
463463
}
464464
void moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) override {
465465
moveAllocationsWithinUMAllocsManagerToGpuDomainCalled++;
466466
}
467467
uint32_t moveAllocationToGpuDomainCalledTimes = 0;
468468
uint32_t moveAllocationsWithinUMAllocsManagerToGpuDomainCalled = 0;
469-
void *migratedAddress = nullptr;
469+
std::vector<void *> migratedAddress;
470470
};
471471

472472
void SetUp() override {
@@ -544,7 +544,7 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests,
544544
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
545545

546546
EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 1u);
547-
EXPECT_EQ(mockPageFaultManager->migratedAddress, ptr);
547+
EXPECT_EQ(mockPageFaultManager->migratedAddress[0], ptr);
548548

549549
mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS;
550550
res = context->evictMemory(device, ptr, size);
@@ -642,7 +642,7 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests,
642642
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
643643

644644
EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 0u);
645-
EXPECT_EQ(mockPageFaultManager->migratedAddress, nullptr);
645+
EXPECT_EQ(mockPageFaultManager->migratedAddress.empty(), true);
646646

647647
mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS;
648648
res = context->evictMemory(device, ptr, size);
@@ -694,8 +694,11 @@ HWTEST_F(ContextMakeMemoryResidentAndMigrationTests,
694694
nullptr, 0, nullptr);
695695
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
696696

697-
EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 1u);
698-
EXPECT_EQ(mockPageFaultManager->migratedAddress, ptr);
697+
EXPECT_EQ(mockPageFaultManager->moveAllocationToGpuDomainCalledTimes, 2u);
698+
699+
if (mockPageFaultManager->migratedAddress[0] != ptr) {
700+
EXPECT_EQ(mockPageFaultManager->migratedAddress[1], ptr);
701+
}
699702

700703
mockMemoryInterface->evictResult = NEO::MemoryOperationsStatus::SUCCESS;
701704
res = context->evictMemory(device, ptr, size);

‎level_zero/core/test/unit_tests/sources/memory/test_memory.cpp‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3033,6 +3033,54 @@ TEST_F(MemoryTest, givenNoDeviceWhenAllocatingSharedMemoryThenDeviceInAllocation
30333033
ASSERT_EQ(result, ZE_RESULT_SUCCESS);
30343034
}
30353035

3036+
TEST_F(MemoryTest, givenCallToMakeMemoryResidentWithInvalidPointerThenInvalidArgumentIsReturned) {
3037+
void *ptr = nullptr;
3038+
ze_result_t res = driverHandle->makeMemoryResident(device, ptr, 1);
3039+
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);
3040+
}
3041+
3042+
TEST_F(MemoryTest,
3043+
givenCallToMakeMemoryResidentWithDeviceMemoryThenAllocationIsNotAddedToVectorOfResidentAllocations) {
3044+
const size_t size = 4096;
3045+
void *ptr = nullptr;
3046+
ze_device_mem_alloc_desc_t deviceDesc = {};
3047+
ze_result_t res = context->allocDeviceMem(device->toHandle(),
3048+
&deviceDesc,
3049+
size,
3050+
0,
3051+
&ptr);
3052+
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
3053+
3054+
DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(context->getDriverHandle());
3055+
size_t previousSize = driverHandleImp->sharedMakeResidentAllocations.size();
3056+
3057+
res = driverHandle->makeMemoryResident(device, ptr, size);
3058+
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
3059+
3060+
size_t currentSize = driverHandleImp->sharedMakeResidentAllocations.size();
3061+
EXPECT_EQ(previousSize, currentSize);
3062+
3063+
context->freeMem(ptr);
3064+
}
3065+
3066+
TEST_F(MemoryTest,
3067+
givenCallToMakeMemoryResidentWithHeapPointerThenSuccessIsReturned) {
3068+
size_t size = 4 * MemoryConstants::pageSize;
3069+
void *ptr = driverHandle->getMemoryManager()->allocateSystemMemory(size, MemoryConstants::pageSize);
3070+
ASSERT_NE(nullptr, ptr);
3071+
3072+
ze_result_t res = driverHandle->importExternalPointer(ptr, MemoryConstants::pageSize);
3073+
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
3074+
3075+
res = driverHandle->makeMemoryResident(device, ptr, MemoryConstants::pageSize);
3076+
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
3077+
3078+
res = driverHandle->releaseImportedPointer(ptr);
3079+
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
3080+
3081+
driverHandle->getMemoryManager()->freeSystemMemory(ptr);
3082+
}
3083+
30363084
TEST_F(MemoryTest, givenCallToCheckMemoryAccessFromDeviceWithInvalidPointerThenInvalidArgumentIsReturned) {
30373085
void *ptr = nullptr;
30383086
ze_result_t res = driverHandle->checkMemoryAccessFromDevice(device, ptr);

0 commit comments

Comments
 (0)