From 784472de77a7e83969e4ba360c95f606b0d7eea0 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Tue, 4 Nov 2025 11:33:20 -0800 Subject: [PATCH 01/19] Streamlined ETL ULT tests Updated code to have a single test run path that all tests use to remove test setup and execution code found in every test. --- .../PresentMonAPI2Tests/EtlTests.cpp | 1860 ++--------------- 1 file changed, 205 insertions(+), 1655 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 87c76982..fc310f74 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -9,15 +9,77 @@ #include "CsvHelper.h" #include "../PresentMonAPI2Loader/Loader.h" #include "../CommonUtilities/pipe/Pipe.h" +#include "../CommonUtilities/str/String.h" #include #include #include +#include using namespace Microsoft::VisualStudio::CppUnitTestFramework; namespace bp = boost::process::v1; +namespace fs = std::filesystem; namespace EtlTests { + static constexpr const char* controlPipe_ = R"(\\.\pipe\pm-etlults-ctrl)"; + static constexpr const char* introNsm_ = "pm_etlults_test_intro"; + static constexpr const char* nsmPrefix_ = "pmon_nsm_utest_"; + + // Necessary data for each test case + struct TestCaseData { + std::string testName; + uint32_t processId; + std::string processName; + std::string etlFile; + std::wstring goldCsvFile; + bool isExpectedFailure; + std::string failureReason; + }; + + static const TestCaseData GOLD_TEST_CASES[] { + {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, + {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, + {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_5192", 5192, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", false, ""}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", false, ""}, + {"test_case_10_42132", 42132, "Marvel-Win64-Shipping.exe", "test_case_10.etl", L"test_case_10.csv", false, ""}, + {"test_case_11_12524", 12524, "Cyberpunk2077.exe", "test_case_11.etl", L"test_case_11.csv", false, ""}, + {"test_case_12_24412", 24412, "Marvel-Win64-Shipping.exe", "test_case_12.etl", L"test_case_12.csv", false, ""}, + }; + + TestCaseData* FindTestCaseByName(const std::string& testName) { + auto it = std::ranges::find_if(GOLD_TEST_CASES, + [&testName](const TestCaseData& tc) { return tc.testName == testName; }); + return it != std::end(GOLD_TEST_CASES) ? const_cast(&*it) : nullptr; + } + void RunTestCaseV2(std::unique_ptr&& pSession, const uint32_t& processId, const std::string& processName, CsvParser& goldCsvFile, std::optional& debugCsvFile) { @@ -95,6 +157,97 @@ namespace EtlTests TEST_CLASS(GoldEtlCsvTests) { std::optional oChild; + private: + std::optional GetAdditionalTestLocation() { + // Check for additional test directory from environment variable + // This allows developers to specify their own test directories without + // modifying the source code. Set PRESENTMON_ADDITIONAL_TEST_DIR environment + // variable or use a .runsettings.user file (see template). + std::wstring additionalTestDir; + wchar_t* envTestDir = nullptr; + size_t envTestDirLen = 0; + if (_wdupenv_s(&envTestDir, &envTestDirLen, L"PRESENTMON_ADDITIONAL_TEST_DIR") == 0 && envTestDir != nullptr) { + additionalTestDir = envTestDir; + free(envTestDir); + } + return additionalTestDir.empty() ? std::nullopt : std::make_optional(pmon::util::str::ToNarrow(additionalTestDir)); + } + + bool SetupTestEnvironment(const std::string& etlFile, const std::string& timedStop, std::unique_ptr& outSession) + { + using namespace std::string_literals; + using namespace std::chrono_literals; + + bp::ipstream out; + bp::opstream in; + + oChild.emplace("PresentMonService.exe"s, + "--timed-stop"s, timedStop, + "--control-pipe"s, controlPipe_, + "--nsm-prefix"s, nsmPrefix_, + "--intro-nsm"s, introNsm_, + "--etl-test-file"s, etlFile, + bp::std_out > out, bp::std_in < in); + + if (!pmon::util::pipe::DuplexPipe::WaitForAvailability(std::string(controlPipe_) + "-in", 500)) { + Assert::Fail(L"Timeout waiting for service control pipe"); + return false; + } + + try { + pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); + pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); + outSession = std::make_unique(controlPipe_); + return true; + } + catch (const std::exception& e) { + std::cout << "Error: " << e.what() << std::endl; + Assert::Fail(L"Failed to connect to service via named pipe"); + return false; + } + } + void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional debugCsv) + { + using namespace std::string_literals; + + if (testCase.isExpectedFailure) { + Assert::Fail(pmon::util::str::ToWide(testCase.failureReason).c_str()); + return; + } + + fs::path etlFile = fs::path(goldPath) / testCase.etlFile; + fs::path csvPath = fs::path(goldPath) / testCase.goldCsvFile; + + CsvParser goldCsvFile; + if (!goldCsvFile.Open(csvPath.wstring(), testCase.processId)) { + Assert::Fail(L"Failed to open gold CSV file"); + return; + } + + std::unique_ptr pSession; + if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { + goldCsvFile.Close(); + return; + } + + RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, + goldCsvFile, debugCsv); + + goldCsvFile.Close(); + } + void RunTestFromCase(const std::string& caseName, bool useDefault = true) + { + auto testCase = FindTestCaseByName(caseName); + if (!testCase) { + Assert::Fail(L"Test case not found"); + return; + } + + fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" + : fs::path(GetAdditionalTestLocation().value_or("")); + + RunGoldCsvTest(*testCase, path.string(), std::nullopt); + } public: TEST_METHOD_CLEANUP(Cleanup) { @@ -107,7 +260,6 @@ namespace EtlTests using namespace std::literals; std::this_thread::sleep_for(50ms); } - TEST_METHOD(OpenCsvTest) { const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; @@ -143,41 +295,11 @@ namespace EtlTests } TEST_METHOD(OpenMockSessionTest) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - + auto testCase = FindTestCaseByName("test_case_0_10792"); + auto etlFile = "..\\..\\tests\\gold\\" + testCase->etlFile; std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } + auto result = SetupTestEnvironment(etlFile, "10000", pSession); + Assert::IsTrue(result, L"SetupTestEnvironment failed"); } TEST_METHOD(ConsumeBlobsTest) @@ -253,395 +375,36 @@ namespace EtlTests } TEST_METHOD(Tc000v2Presenter10792) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10792; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_10792"); } TEST_METHOD(Tc000v2DWM1268) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1268; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_1268"); } + TEST_METHOD(Tc000v2Presenter8320) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 8320; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_8320"); } TEST_METHOD(Tc000v2Presenter11648) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11648; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11648"); } TEST_METHOD(Tc000v2Presenter3976) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 3976; - const std::string processName = "Presenter.exe"; - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - std::optional debugCsv; // Empty optional - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_3976"); } TEST_METHOD(Tc000v2Presenter11112) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11112; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11112"); } TEST_METHOD(Tc000v2Presenter2032) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 2032; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_2032"); } TEST_METHOD(Tc000v2Presenter5988) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5988; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_5988"); } TEST_METHOD(Tc000v2Presenter12268) { @@ -652,597 +415,52 @@ namespace EtlTests // the process is not active and exit. Need to add some type of synchronization // in mock presentmon session to not shutdown the session until notified // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12268; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_12268"); } TEST_METHOD(Tc000v2Presenter11100) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11100; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11100"); } TEST_METHOD(Tc001v2Dwm1564) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // This test is an expected failure. The reason for the failure is the - // mock presentmon session is writing a present to the nsm that is - // earlier than the console application allows because of swap chain - // initialization that is not implemented in the mock presentmon session. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1564; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_1564"); } TEST_METHOD(Tc001v2Presenter24560) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24560; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_24560"); } TEST_METHOD(Tc001v2devenv24944) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24944; - const std::string processName = "devenv.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_24944"); } TEST_METHOD(Tc002v2Dwm1300) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1300; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_1300"); } TEST_METHOD(Tc002v2Presenter10016) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10016; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_10016"); } TEST_METHOD(Tc002v2Presenter5348) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5348; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_5348"); } TEST_METHOD(Tc002v2Presenter5220) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5220; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_5220"); } TEST_METHOD(Tc003v2Dwm1252) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1252; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_1252"); } TEST_METHOD(Tc003v2Presenter5892) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5892; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_5892"); } TEST_METHOD(Tc003v2Presenter10112) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10112; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_10112"); } TEST_METHOD(Tc003v2Presenter12980) { @@ -1253,201 +471,19 @@ namespace EtlTests // the process is not active and exit. Need to add some type of synchronization // in mock presentmon session to not shutdown the session until notified // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12980; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_12980"); } - TEST_METHOD(Tc004v2Presenter5192) + TEST_METHOD(Tc003v2Presenter5192) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5192; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_5192"); } TEST_METHOD(Tc004v2Presenter5236) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5236; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_4_5236"); } TEST_METHOD(Tc004v2Presenter8536) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 8536; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_4_8536"); } TEST_METHOD(Tc004v2Presenter9620) { @@ -1455,532 +491,46 @@ namespace EtlTests // finishedby the the mock presentmon session. If the ETL session finishes // and sets the process id to not active from the mock presentmon session // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 9620; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc004v2Dwm10376) - { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10376; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc005v2PresentBench24892) - { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24892; - const std::string processName = "PresentBench.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_5.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_5.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc006v2CPXellOn10796Ext) - { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10796; - const std::string processName = "cpLauncher.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_6.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_6.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + // the process is not active and exit. Need to add some type of synchronization + // in mock presentmon session to not shutdown the session until notified + // by close session call. + RunTestFromCase("test_case_4_9620"); + } + TEST_METHOD(Tc004v2Dwm10376) + { + RunTestFromCase("test_case_4_10376"); + } + TEST_METHOD(Tc005v2PresentBench24892) + { + RunTestFromCase("test_case_5_24892"); + } + TEST_METHOD(Tc006v2CPXellOn10796Ext) + { + RunTestFromCase("test_case_6_10796", false); } TEST_METHOD(Tc007v2CPXellOnFgOn11320Ext) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11320; - const std::string processName = "cpLauncher.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_7.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_7.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_7_11320", false); } TEST_METHOD(Tc008v2ACSXellOnFgOn6920Ext) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // This test is an expected failure. The reason for the failure is the - // mock presentmon session is writing a present to the nsm that is - // earlier than the console application allows because of swap chain - // initialization that is not implemented in the mock presentmon session. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 6920; - const std::string processName = "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_8.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_8.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_8_6920", false); } TEST_METHOD(Tc009v2F124XellOnFgOn10340Ext) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10340; - const std::string processName = "F1_24.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_9.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_9.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_9_10340", false); } TEST_METHOD(Tc010MarvelOnNvPcl1FgOnExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 42132; - const std::string processName = "Marvel-Win64-Shipping.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_10.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_10.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_10_42132", false); } TEST_METHOD(Tc011CP2077Pcl2FgOffRelexOffExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12524; - const std::string processName = "Cyberpunk2077.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_11.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_11.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - std::this_thread::sleep_for(1000ms); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_11_12524", false); } TEST_METHOD(Tc012MarvelOnNvPcl3FgOnAutoReflexOnFrameDelayExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24412; - const std::string processName = "Marvel-Win64-Shipping.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_12.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_12.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - std::string folder = "F:\\EtlTesting\\ETLDebugging\\testcase12\\"s; - std::string csvName = "debug.csv"s; - debugCsv = CreateCsvFile(folder,csvName); - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - std::this_thread::sleep_for(1000ms); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_12_24412", false); } }; } \ No newline at end of file From fdb35ba946281eaab0430364698192c907f0fdcf Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Tue, 4 Nov 2025 11:35:25 -0800 Subject: [PATCH 02/19] Removed now redundant test ConsumeBlobs is no longer needed as every test case now uses the same path that was used by ConsumeBlobs --- .../PresentMonAPI2Tests/EtlTests.cpp | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index fc310f74..fe275d62 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -301,78 +301,6 @@ namespace EtlTests auto result = SetupTestEnvironment(etlFile, "10000", pSession); Assert::IsTrue(result, L"SetupTestEnvironment failed"); } - - TEST_METHOD(ConsumeBlobsTest) - { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10792; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName + "-in", 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - pmapi::ProcessTracker processTracker; - uint32_t totalFramesValidated = 0; - - PM_QUERY_ELEMENT queryElements[]{ - { PM_METRIC_BETWEEN_PRESENTS, PM_STAT_NONE, 0, 0}, - }; - - auto frameQuery = pSession->RegisterFrameQuery(queryElements); - auto blobs = frameQuery.MakeBlobContainer(8); - - processTracker = pSession->TrackProcess(processId); - - using Clock = std::chrono::high_resolution_clock; - const auto start = Clock::now(); - - while (1) { - frameQuery.Consume(processTracker, blobs); - if (blobs.GetNumBlobsPopulated() == 0) { - if (Clock::now() - start >= 1s) { - // if it takes longer than 1 second to consume the first frame, throw failure - Assert::Fail(L"Timeout waiting to consume first frame"); - } - std::this_thread::sleep_for(8ms); - } - else { - break; - } - } - } TEST_METHOD(Tc000v2Presenter10792) { RunTestFromCase("test_case_0_10792"); From c372d76780563620d359b0057cead37e42c4069c Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 10:38:27 -0800 Subject: [PATCH 03/19] Better name creation method and filesystem handling for Debug csvs --- .../PresentMonAPI2Tests/CsvHelper.h | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h index 6063da67..e6d6a2be 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h +++ b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h @@ -12,8 +12,10 @@ #include #include #include +#include using namespace Microsoft::VisualStudio::CppUnitTestFramework; +namespace fs = std::filesystem; enum Header { Header_Application, @@ -344,16 +346,31 @@ bool Validate(const T& param1, const T& param2) { } -std::optional CreateCsvFile(std::string& output_dir, std::string& processName) +std::optional CreateCsvFile(const std::string& output_dir, const std::string& processName) { // Setup csv file time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); tm local_time; localtime_s(&local_time, &now); - std::ofstream csvFile; - std::string csvFileName = output_dir + processName; + try { - csvFile.open(csvFileName); + // Use filesystem to construct the path properly + fs::path outputPath(output_dir); + fs::path fileName = processName + "_" + + std::to_string(local_time.tm_year + 1900) + + std::to_string(local_time.tm_mon + 1) + + std::to_string(local_time.tm_mday) + + std::to_string(local_time.tm_hour) + + std::to_string(local_time.tm_min) + + std::to_string(local_time.tm_sec) + ".csv"; + + fs::path fullPath = outputPath / fileName; + + std::ofstream csvFile(fullPath); + if (!csvFile.is_open()) { + return std::nullopt; + } + csvFile << "Application,ProcessID,SwapChainAddress,PresentRuntime" ",SyncInterval,PresentFlags,AllowsTearing,PresentMode" From 17823ea444db54e96b9aaefec30d5ec40737fc1d Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 10:41:04 -0800 Subject: [PATCH 04/19] Fix for ULT ETL testing When testing ETLs through the ULT system at the start the circular buffer can be very small and cause the peek code to wrap around when looking for the next displayed frame. Use the PresentStartTime of the "next" displayed frame to ensure it is in fact after the current frame --- IntelPresentMon/Streamer/StreamClient.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/IntelPresentMon/Streamer/StreamClient.cpp b/IntelPresentMon/Streamer/StreamClient.cpp index 13dd765b..319fa661 100644 --- a/IntelPresentMon/Streamer/StreamClient.cpp +++ b/IntelPresentMon/Streamer/StreamClient.cpp @@ -389,6 +389,19 @@ PM_STATUS StreamClient::ConsumePtrToNextNsmFrameData(const PmNsmFrameData** pNsm *pNextFrame = nullptr; return PM_STATUS::PM_STATUS_SUCCESS; } + + if (pFrameDataOfNextDisplayed != nullptr) { + if ((*pNsmData)->present_event.PresentStartTime >= + (*pFrameDataOfNextDisplayed)->present_event.PresentStartTime) { + // The next displayed frame must be after the current frame. + // This is an error so reset the next_dequeue_idx back to where we first started. + next_dequeue_idx_ = previous_dequeue_idx; + // Also reset the current and next frame data pointers + *pNsmData = nullptr; + *pNextFrame = nullptr; + return PM_STATUS::PM_STATUS_SUCCESS; + } + } PeekPreviousFrames( pFrameDataOfLastPresented, pFrameDataOfLastAppPresented, From 6ac328854c55143e4099f789eb0be5fbf0e2c4a9 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 12:23:12 -0800 Subject: [PATCH 05/19] Updated test cases Added pollCount and waitTimeSecs fields to allow for test cases that have longer lead in times to the targeted application's first frame. Also fixed incorrect naming of various test cases. --- .../PresentMonAPI2Tests/EtlTests.cpp | 126 ++++++++++-------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index fe275d62..4dc911e5 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -32,46 +32,49 @@ namespace EtlTests std::string processName; std::string etlFile; std::wstring goldCsvFile; + int pollCount; + int waitTimeSecs; bool isExpectedFailure; std::string failureReason; }; static const TestCaseData GOLD_TEST_CASES[] { - {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, - {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, - {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_5192", 5192, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", false, ""}, - {"test_case_10_42132", 42132, "Marvel-Win64-Shipping.exe", "test_case_10.etl", L"test_case_10.csv", false, ""}, - {"test_case_11_12524", 12524, "Cyberpunk2077.exe", "test_case_11.etl", L"test_case_11.csv", false, ""}, - {"test_case_12_24412", 24412, "Marvel-Win64-Shipping.exe", "test_case_12.etl", L"test_case_12.csv", false, ""}, + {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, + {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, + {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_4_5192", 5192, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 10, 1, false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_12_10168", 10168, "F1_24.exe", "test_case_12.etl", L"test_case_12.csv", 20, 5, false, ""}, + {"test_case_13_11780", 11780, "Dingo.Main_Win64_retail.exe", "test_case_13.etl", L"test_case_13.csv", 10, 1, false, ""}, }; TestCaseData* FindTestCaseByName(const std::string& testName) { @@ -82,10 +85,10 @@ namespace EtlTests void RunTestCaseV2(std::unique_ptr&& pSession, const uint32_t& processId, const std::string& processName, CsvParser& goldCsvFile, - std::optional& debugCsvFile) { + std::optional& debugCsvFile, int pollCount, int waitTimeSecs) { using namespace std::chrono_literals; pmapi::ProcessTracker processTracker; - static constexpr uint32_t numberOfBlobs = 10000u; + static constexpr uint32_t numberOfBlobs = 2000; uint32_t totalFramesValidated = 0; PM_QUERY_ELEMENT queryElements[]{ @@ -134,12 +137,12 @@ namespace EtlTests frameQuery.Consume(processTracker, blobs); if (blobs.GetNumBlobsPopulated() == 0) { // if we poll 10 times in a row and get no new frames, consider this ETL finished - if (++emptyPollCount >= 10) { + if (++emptyPollCount >= pollCount) { if (totalFramesValidated > 0) { // only finish if we have consumed at least one frame break; } - else if (Clock::now() - start >= 1s) { + else if (Clock::now() - start >= std::chrono::seconds(waitTimeSecs)) { // if it takes longer than 1 second to consume the first frame, throw failure Assert::Fail(L"Timeout waiting to consume first frame"); } @@ -206,7 +209,7 @@ namespace EtlTests return false; } } - void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional debugCsv) + void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv) { using namespace std::string_literals; @@ -231,11 +234,11 @@ namespace EtlTests } RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, - goldCsvFile, debugCsv); + goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); goldCsvFile.Close(); } - void RunTestFromCase(const std::string& caseName, bool useDefault = true) + void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) { auto testCase = FindTestCaseByName(caseName); if (!testCase) { @@ -246,7 +249,16 @@ namespace EtlTests fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" : fs::path(GetAdditionalTestLocation().value_or("")); - RunGoldCsvTest(*testCase, path.string(), std::nullopt); + std::optional debugCsv; + if (createDebugCsv) { + auto outputDir = path.string(); + auto debugCsvName = testCase->processName + "-debug.csv"; + debugCsv = CreateCsvFile(outputDir, debugCsvName); + } + RunGoldCsvTest(*testCase, path.string(), debugCsv); + if (debugCsv.has_value()) { + debugCsv->close(); + } } public: TEST_METHOD_CLEANUP(Cleanup) @@ -401,9 +413,9 @@ namespace EtlTests // by close session call. RunTestFromCase("test_case_3_12980"); } - TEST_METHOD(Tc003v2Presenter5192) + TEST_METHOD(Tc004v2Presenter5192) { - RunTestFromCase("test_case_3_5192"); + RunTestFromCase("test_case_4_5192", true, true); } TEST_METHOD(Tc004v2Presenter5236) { @@ -432,33 +444,37 @@ namespace EtlTests { RunTestFromCase("test_case_5_24892"); } - TEST_METHOD(Tc006v2CPXellOn10796Ext) + TEST_METHOD(Tc006CP2077) { RunTestFromCase("test_case_6_10796", false); } - TEST_METHOD(Tc007v2CPXellOnFgOn11320Ext) + TEST_METHOD(Tc007CP2077) { RunTestFromCase("test_case_7_11320", false); } - TEST_METHOD(Tc008v2ACSXellOnFgOn6920Ext) + TEST_METHOD(Tc008ACShadows) { RunTestFromCase("test_case_8_6920", false); } - TEST_METHOD(Tc009v2F124XellOnFgOn10340Ext) + TEST_METHOD(Tc009F124) { RunTestFromCase("test_case_9_10340", false); } - TEST_METHOD(Tc010MarvelOnNvPcl1FgOnExt) + TEST_METHOD(Tc010NarakaBladepoint) + { + RunTestFromCase("test_case_10_9888", false); + } + TEST_METHOD(Tc011NarakaBladepoint) { - RunTestFromCase("test_case_10_42132", false); + RunTestFromCase("test_case_11_1524", false); } - TEST_METHOD(Tc011CP2077Pcl2FgOffRelexOffExt) + TEST_METHOD(Tc012F124) { - RunTestFromCase("test_case_11_12524", false); + RunTestFromCase("test_case_12_10168", false); } - TEST_METHOD(Tc012MarvelOnNvPcl3FgOnAutoReflexOnFrameDelayExt) + TEST_METHOD(Tc013Dingo) { - RunTestFromCase("test_case_12_24412", false); + RunTestFromCase("test_case_13_11780", false); } }; } \ No newline at end of file From 103d0d80d9fcdb5b443c53ffd162a165ad3d2077 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Fri, 7 Nov 2025 09:29:37 -0800 Subject: [PATCH 06/19] Updated CsvHelper to use exceptions instead of MSTest Assert Provides better logging when failures occur. --- .../PresentMonAPI2Tests/CsvHelper.h | 302 ++++++++---------- .../PresentMonAPI2Tests/EtlTests.cpp | 104 +++++- 2 files changed, 229 insertions(+), 177 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h index e6d6a2be..bdbb2632 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h +++ b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h @@ -178,6 +178,63 @@ std::wstring CreateErrorString(Header columnId, size_t line) return errorMessage; } +class CsvException : public std::runtime_error { +public: + explicit CsvException(const std::string& msg) : std::runtime_error(msg) {} +}; + +class CsvConversionException : public CsvException { + Header columnId_; + size_t line_; + std::string value_; +public: + CsvConversionException(Header columnId, size_t line, const std::string& value) + : CsvException(std::format("Invalid {} at line {}: '{}'", + GetHeaderString(columnId), line, value)) + , columnId_(columnId) + , line_(line) + , value_(value) + { + } + + Header GetColumnId() const { return columnId_; } + size_t GetLine() const { return line_; } + const std::string& GetValue() const { return value_; } +}; + +class CsvFileException : public CsvException { +public: + explicit CsvFileException(const std::string& msg) : CsvException(msg) {} +}; + +class CsvValidationException : public CsvException { + Header columnId_; + size_t line_; + + // Helper to format values for error messages + template + static auto FormatValue(const T& value) { + if constexpr (std::is_enum_v) { + return static_cast>(value); + } else { + return value; + } + } + +public: + template + CsvValidationException(Header columnId, size_t line, const T& receivedValue, const T& expectedValue) + : CsvException(std::format("{} at line {}: Do not match. Received value {}, Expected value {}", + GetHeaderString(columnId), line, FormatValue(receivedValue), FormatValue(expectedValue))) + , columnId_(columnId) + , line_(line) + { + } + + Header GetColumnId() const { return columnId_; } + size_t GetLine() const { return line_; } +}; + template class CharConvert { public: @@ -192,7 +249,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stod(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -201,7 +258,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stoul(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -210,7 +267,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stoi(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -230,7 +287,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = static_cast(result1); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } @@ -245,7 +302,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_GRAPHICS_RUNTIME_UNKNOWN; } else { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } @@ -275,7 +332,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_PRESENT_MODE_UNKNOWN; } else { - Assert::Fail(CreateErrorString(Header_PresentMode, line).c_str()); + throw CsvConversionException(Header_PresentMode, line, data); } } else if constexpr (std::is_same::value) { @@ -298,12 +355,12 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_FRAME_TYPE_INTEL_XEFG; } else { - Assert::Fail(CreateErrorString(Header_FrameType, line).c_str()); + throw CsvConversionException(Header_FrameType, line, data); } } else { - Assert::Fail(CreateErrorString(UnknownHeader, line).c_str()); + throw CsvException("Unknown type conversion requested"); } } @@ -326,6 +383,44 @@ size_t countDecimalPlaces(double value) { return count; } +template +void ValidateOrThrow(Header columnId, size_t line, const T& received, const T& expected) { + if constexpr (std::is_same_v) { + // Both NaN is considered a match (both represent missing/invalid data) + if (std::isnan(received) && std::isnan(expected)) { + return; + } + // One NaN and one value is a mismatch + if (std::isnan(received) || std::isnan(expected)) { + throw CsvValidationException(columnId, line, received, expected); + } + // Both are valid numbers, compare with threshold + auto min = std::min(countDecimalPlaces(received), countDecimalPlaces(expected)); + double threshold = pow(0.1, min); + double difference = received - expected; + if (difference <= -threshold || difference >= threshold) { + throw CsvValidationException(columnId, line, received, expected); + } + } + else { + if (received != expected) { + throw CsvValidationException(columnId, line, received, expected); + } + } +} + +// Overload for std::optional - treats empty optional as NaN +void ValidateOrThrow(Header columnId, size_t line, const std::optional& received, double expected) { + if (received.has_value()) { + ValidateOrThrow(columnId, line, received.value(), expected); + } else { + // No value in CSV (optional is empty), expected should be NaN + if (!std::isnan(expected)) { + throw CsvValidationException(columnId, line, std::nan(""), expected); + } + } +} + template bool Validate(const T& param1, const T& param2) { if constexpr (std::is_same::value) { @@ -605,209 +700,93 @@ bool CsvParser::VerifyBlobAgainstCsv(const std::string& processName, const unsig // Go through all of the active headers and validate results for (const auto& pair : activeColHeadersMap_) { - - bool columnsMatch = false; switch (pair.second) { case Header_Application: - columnsMatch = true; + // Skip validation for application name break; case Header_ProcessID: - columnsMatch = Validate(v2MetricRow_.processId, processId_); + ValidateOrThrow(pair.second, line_, v2MetricRow_.processId, processId_); break; case Header_SwapChainAddress: - columnsMatch = Validate(v2MetricRow_.swapChain, swapChain); + ValidateOrThrow(pair.second, line_, v2MetricRow_.swapChain, swapChain); break; case Header_Runtime: - columnsMatch = Validate(v2MetricRow_.runtime, graphicsRuntime); + ValidateOrThrow(pair.second, line_, v2MetricRow_.runtime, graphicsRuntime); break; case Header_SyncInterval: - columnsMatch = Validate(v2MetricRow_.syncInterval, syncInterval); + ValidateOrThrow(pair.second, line_, v2MetricRow_.syncInterval, syncInterval); break; case Header_PresentFlags: - columnsMatch = Validate(v2MetricRow_.presentFlags, presentFlags); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentFlags, presentFlags); break; case Header_AllowsTearing: - columnsMatch = Validate(v2MetricRow_.allowsTearing, (uint32_t)allowsTearing); + ValidateOrThrow(pair.second, line_, v2MetricRow_.allowsTearing, (uint32_t)allowsTearing); break; case Header_PresentMode: - columnsMatch = Validate(v2MetricRow_.presentMode, presentMode); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentMode, presentMode); break; case Header_FrameType: - columnsMatch = Validate(v2MetricRow_.frameType, frameType); + ValidateOrThrow(pair.second, line_, v2MetricRow_.frameType, frameType); break; case Header_TimeInSeconds: - columnsMatch = Validate(v2MetricRow_.presentStartQPC, timeQpc); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentStartQPC, timeQpc); break; case Header_CPUStartQPC: - columnsMatch = Validate(v2MetricRow_.cpuFrameQpc, cpuStartQpc); + ValidateOrThrow(pair.second, line_, v2MetricRow_.cpuFrameQpc, cpuStartQpc); break; case Header_MsBetweenAppStart: - columnsMatch = Validate(v2MetricRow_.msBetweenAppStart, msBetweenAppStart); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenAppStart, msBetweenAppStart); break; case Header_MsCPUBusy: - columnsMatch = Validate(v2MetricRow_.msCpuBusy, msCpuBusy); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msCpuBusy, msCpuBusy); break; case Header_MsCPUWait: - columnsMatch = Validate(v2MetricRow_.msCpuWait, msCpuWait); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msCpuWait, msCpuWait); break; case Header_MsGPULatency: - columnsMatch = Validate(v2MetricRow_.msGpuLatency, msGpuLatency); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuLatency, msGpuLatency); break; case Header_MsGPUTime: - columnsMatch = Validate(v2MetricRow_.msGpuTime, msGpuTime); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuTime, msGpuTime); break; case Header_MsGPUBusy: - columnsMatch = Validate(v2MetricRow_.msGpuBusy, msGpuBusy); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuBusy, msGpuBusy); break; case Header_MsGPUWait: - columnsMatch = Validate(v2MetricRow_.msGpuWait, msGpuWait); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuWait, msGpuWait); break; case Header_MsBetweenSimulationStart: - if (v2MetricRow_.msBetweenSimStart.has_value()) { - columnsMatch = Validate(v2MetricRow_.msBetweenSimStart.value(), msBetweenSimStartTime); - } - else - { - if (std::isnan(msBetweenSimStartTime)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenSimStart, msBetweenSimStartTime); break; case Header_MsUntilDisplayed: - if (v2MetricRow_.msUntilDisplayed.has_value()) { - columnsMatch = Validate(v2MetricRow_.msUntilDisplayed.value(), msUntilDisplayed); - } - else - { - if (std::isnan(msUntilDisplayed)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msUntilDisplayed, msUntilDisplayed); break; case Header_MsBetweenDisplayChange: - if (v2MetricRow_.msBetweenDisplayChange.has_value()) { - columnsMatch = Validate(v2MetricRow_.msBetweenDisplayChange.value(), msBetweenDisplayChange); - } - else - { - if (std::isnan(msBetweenDisplayChange)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenDisplayChange, msBetweenDisplayChange); break; case Header_MsPCLatency: - if (v2MetricRow_.msPcLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msPcLatency.value(), msPcLatency); - } - else - { - if (std::isnan(msPcLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msPcLatency, msPcLatency); break; case Header_MsAnimationError: - if (v2MetricRow_.msAnimationError.has_value()) { - columnsMatch = Validate(v2MetricRow_.msAnimationError.value(), msAnimationError); - } - else - { - if (std::isnan(msAnimationError)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msAnimationError, msAnimationError); break; case Header_AnimationTime: - if (v2MetricRow_.animationTime.has_value()) { - columnsMatch = Validate(v2MetricRow_.animationTime.value(), animationTime); - } - else - { - if (std::isnan(animationTime)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.animationTime, animationTime); break; case Header_MsClickToPhotonLatency: - if (v2MetricRow_.msClickToPhotonLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msClickToPhotonLatency.value(), msClickToPhotonLatency); - } - else - { - if (std::isnan(msClickToPhotonLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msClickToPhotonLatency, msClickToPhotonLatency); break; case Header_MsAllInputToPhotonLatency: - if (v2MetricRow_.msAllInputToPhotonLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msAllInputToPhotonLatency.value(), msAllInputToPhotonLatency); - } - else - { - if (std::isnan(msAllInputToPhotonLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msAllInputToPhotonLatency, msAllInputToPhotonLatency); break; case Header_MsInstrumentedLatency: - if (v2MetricRow_.msInstrumentedLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msInstrumentedLatency.value(), msInstrumentedLatency); - } - else - { - if (std::isnan(msInstrumentedLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msInstrumentedLatency, msInstrumentedLatency); break; default: - columnsMatch = true; + // Unknown header, skip validation break; } - if (columnsMatch == false) { - // If the columns do not match, create an error string - // and assert failure. - Assert::Fail(CreateErrorString(pair.second, line_).c_str()); - } - Assert::IsTrue(columnsMatch, CreateErrorString(pair.second, line_).c_str()); } } @@ -854,7 +833,8 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { cols_.clear(); if (_wfopen_s(&fp_, path.c_str(), L"r")) { - return false; + throw CsvFileException(std::format("Failed to open CSV file: {}", + pmon::util::str::ToNarrow(path))); } // Remove UTF-8 marker if there is one. @@ -880,9 +860,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { default: if ((size_t)h < KnownHeaderCount) { if (headerColumnIndex_[(size_t)h] != SIZE_MAX) { - std::wstring errorMessage = L"Duplicate column: "; - errorMessage += pmon::util::str::ToWide(cols_[i]); - Assert::Fail(errorMessage.c_str()); + throw CsvFileException(std::format("Duplication column: {}", cols_[i])); } else { headerColumnIndex_[(size_t)h] = i; @@ -890,8 +868,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { break; } else { - std::wstring errorMessage = L"Index outside of known headers."; - Assert::Fail(errorMessage.c_str()); + throw CsvFileException("Index outside of known headers"); } } } @@ -925,7 +902,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { Header_MsPCLatency}); if (!columnsOK) { - Assert::Fail(L"Missing required columns"); + throw CsvFileException("Missing required columns in CSV file"); } // Create a vector of active headers to be used when reading @@ -1213,7 +1190,7 @@ void CsvParser::ConvertToMetricDataType(const char* data, Header columnId) } break; default: - Assert::Fail(CreateErrorString(UnknownHeader, line_).c_str()); + throw CsvConversionException(UnknownHeader, line_, data); } return; @@ -1227,9 +1204,8 @@ bool CsvParser::ReadRow(bool gatherMetrics) // Read a line if (fgets(row_, _countof(row_), fp_) == nullptr) { if (ferror(fp_) != 0) { - std::wstring errorMessage = L"File read error at line: "; - errorMessage += std::to_wstring(line_); - Assert::Fail(errorMessage.c_str()); + throw CsvFileException(std::format("File read error at line: {}", + std::to_string(line_))); } return false; } diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 4dc911e5..97293fdb 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2022-2023 Intel Corporation +// Copyright (C) 2022-2023 Intel Corporation // SPDX-License-Identifier: MIT #include "../CommonUtilities/win/WinAPI.h" #include @@ -49,7 +49,7 @@ namespace EtlTests {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, @@ -68,8 +68,8 @@ namespace EtlTests {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 10, 1, false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 20, 5, false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 20, 5, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, @@ -213,12 +213,7 @@ namespace EtlTests { using namespace std::string_literals; - if (testCase.isExpectedFailure) { - Assert::Fail(pmon::util::str::ToWide(testCase.failureReason).c_str()); - return; - } - - fs::path etlFile = fs::path(goldPath) / testCase.etlFile; + fs::path etlFile = fs::path(goldPath) / testCase.etlFile; fs::path csvPath = fs::path(goldPath) / testCase.goldCsvFile; CsvParser goldCsvFile; @@ -230,14 +225,95 @@ namespace EtlTests std::unique_ptr pSession; if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { goldCsvFile.Close(); + Assert::Fail(L"Failed to setup test environment"); return; } - RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, - goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); + // Track if we encountered an assertion failure + bool testPassed = true; + std::string exceptionMessage; + + try { + RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, + goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); + } + catch (const CsvValidationException& e) { + testPassed = false; + exceptionMessage = std::format( + "CSV Validation Error:\n" + " Column: {}\n" + " Line: {}\n" + " Details: {}", + GetHeaderString(e.GetColumnId()), + e.GetLine(), + e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvConversionException& e) { + testPassed = false; + exceptionMessage = std::format( + "CSV Conversion Error:\n" + " Column: {}\n" + " Line: {}\n" + " Invalid Value: '{}'\n" + " Details: {}", + GetHeaderString(e.GetColumnId()), + e.GetLine(), + e.GetValue(), + e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvFileException& e) { + testPassed = false; + exceptionMessage = std::format("CSV File Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvException& e) { + testPassed = false; + exceptionMessage = std::format("CSV Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const std::exception& e) { + testPassed = false; + exceptionMessage = std::format("Unexpected Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (...) { + testPassed = false; + exceptionMessage = "Unknown exception caught"; + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } goldCsvFile.Close(); + + // Now handle expected failure logic + if (testCase.isExpectedFailure) { + if (testPassed) { + // Test passed but was expected to fail - this is noteworthy! + Logger::WriteMessage(std::format( + "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", + testCase.testName, testCase.failureReason).c_str()); + // Still pass the test since it's good news + } + else { + // Test failed as expected + Logger::WriteMessage(std::format( + "[FAIL] Expected failure: {}\n", + testCase.failureReason).c_str()); + // Re-throw to fail the test, but it's documented as expected + Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", + pmon::util::str::ToWide(testCase.failureReason)).c_str()); + } + } + else if (!testPassed) { + // Unexpected failure - fail the test with detailed message + Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + } + // else: test passed and wasn't expected to fail - all good! } + void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) { auto testCase = FindTestCaseByName(caseName); @@ -364,7 +440,7 @@ namespace EtlTests TEST_METHOD(Tc001v2Dwm1564) { - RunTestFromCase("test_case_1_1564"); + RunTestFromCase("test_case_1_1564", true, true); } TEST_METHOD(Tc001v2Presenter24560) { @@ -415,7 +491,7 @@ namespace EtlTests } TEST_METHOD(Tc004v2Presenter5192) { - RunTestFromCase("test_case_4_5192", true, true); + RunTestFromCase("test_case_4_5192"); } TEST_METHOD(Tc004v2Presenter5236) { From 2cc928dcfd8775ee3e26713839d10b57d7aa5a7b Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Mon, 10 Nov 2025 09:54:10 -0800 Subject: [PATCH 07/19] Removed single test run support for IPM ETL ULTs --- .../PresentMonAPI2Tests/EtlTests.cpp | 551 ++++++++++-------- 1 file changed, 306 insertions(+), 245 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 97293fdb..db8f0fe0 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -25,7 +25,7 @@ namespace EtlTests static constexpr const char* introNsm_ = "pm_etlults_test_intro"; static constexpr const char* nsmPrefix_ = "pmon_nsm_utest_"; - // Necessary data for each test case + // Test case data structure - loaded from CSV file struct TestCaseData { std::string testName; uint32_t processId; @@ -36,51 +36,125 @@ namespace EtlTests int waitTimeSecs; bool isExpectedFailure; std::string failureReason; + bool useAdditionalTestLocation; // Load from additional test directory (runsettings) + bool produceDebugCsv; // Generate debug CSV output + bool runTest; // Whether to run this test (for selective debugging) }; - static const TestCaseData GOLD_TEST_CASES[] { - {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_4_5192", 5192, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 20, 5, false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 20, 5, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_12_10168", 10168, "F1_24.exe", "test_case_12.etl", L"test_case_12.csv", 20, 5, false, ""}, - {"test_case_13_11780", 11780, "Dingo.Main_Win64_retail.exe", "test_case_13.etl", L"test_case_13.csv", 10, 1, false, ""}, - }; + // Helper function to parse boolean from CSV string + bool ParseBool(const std::string& value) { + std::string lower = value; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + return lower == "true" || lower == "1" || lower == "yes" || lower == "y"; + } - TestCaseData* FindTestCaseByName(const std::string& testName) { - auto it = std::ranges::find_if(GOLD_TEST_CASES, - [&testName](const TestCaseData& tc) { return tc.testName == testName; }); - return it != std::end(GOLD_TEST_CASES) ? const_cast(&*it) : nullptr; + // Helper function to trim whitespace from string + std::string Trim(const std::string& str) { + const auto start = str.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return ""; + const auto end = str.find_last_not_of(" \t\r\n"); + return str.substr(start, end - start + 1); + } + + // Parse a CSV line handling quoted fields + std::vector ParseCsvLine(const std::string& line) { + std::vector fields; + std::string field; + bool inQuotes = false; + + for (size_t i = 0; i < line.length(); ++i) { + char c = line[i]; + + if (c == '"') { + inQuotes = !inQuotes; + } + else if (c == ',' && !inQuotes) { + fields.push_back(Trim(field)); + field.clear(); + } + else { + field += c; + } + } + fields.push_back(Trim(field)); + return fields; + } + + // Load test cases from CSV file + // CSV Format: TestName,ProcessID,ProcessName,EtlFile,GoldCsvFile,PollCount,WaitTimeSecs,IsExpectedFailure,FailureReason,UseAdditionalTestLocation,ProduceDebugCsv,RunTest + std::vector LoadTestCasesFromCsv(const std::string& csvFilePath) { + std::vector testCases; + + // Convert to absolute path for better error reporting + fs::path absolutePath = fs::absolute(csvFilePath); + + std::ifstream file(csvFilePath); + if (!file.is_open()) { + throw std::runtime_error(std::format( + "Failed to open test cases CSV file:\n" + " Requested path: {}\n" + " Absolute path: {}\n" + " Current directory: {}", + csvFilePath, + absolutePath.string(), + fs::current_path().string())); + } + + std::string line; + bool isFirstLine = true; + size_t lineNumber = 0; + + while (std::getline(file, line)) { + lineNumber++; + + // Skip empty lines + if (Trim(line).empty()) { + continue; + } + + // Skip header line + if (isFirstLine) { + isFirstLine = false; + continue; + } + + try { + auto fields = ParseCsvLine(line); + + // Validate field count + if (fields.size() < 12) { + throw std::runtime_error(std::format( + "Line {}: Expected at least 12 fields, got {}", + lineNumber, fields.size())); + } + + TestCaseData testCase; + testCase.testName = fields[0]; + testCase.processId = std::stoul(fields[1]); + testCase.processName = fields[2]; + testCase.etlFile = fields[3]; + testCase.goldCsvFile = pmon::util::str::ToWide(fields[4]); + testCase.pollCount = std::stoi(fields[5]); + testCase.waitTimeSecs = std::stoi(fields[6]); + testCase.isExpectedFailure = ParseBool(fields[7]); + testCase.failureReason = fields[8]; + testCase.useAdditionalTestLocation = ParseBool(fields[9]); + testCase.produceDebugCsv = ParseBool(fields[10]); + testCase.runTest = ParseBool(fields[11]); + + testCases.push_back(testCase); + } + catch (const std::exception& e) { + throw std::runtime_error(std::format( + "Error parsing line {}: {}", lineNumber, e.what())); + } + } + + if (testCases.empty()) { + throw std::runtime_error("No test cases loaded from CSV file"); + } + + return testCases; } void RunTestCaseV2(std::unique_ptr&& pSession, @@ -143,8 +217,8 @@ namespace EtlTests break; } else if (Clock::now() - start >= std::chrono::seconds(waitTimeSecs)) { - // if it takes longer than 1 second to consume the first frame, throw failure - Assert::Fail(L"Timeout waiting to consume first frame"); + // if it takes longer than alloted test time to consume the first frame, throw failure + throw CsvException("Timeout waiting to consume first frame"); } } std::this_thread::sleep_for(8ms); @@ -209,7 +283,10 @@ namespace EtlTests return false; } } - void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv) + // Returns true if test passed, false if test failed + // When throwOnFailure=false, returns status instead of throwing + // When throwOnFailure=true, throws Assert::Fail on failure + bool RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv, bool throwOnFailure = true) { using namespace std::string_literals; @@ -218,15 +295,25 @@ namespace EtlTests CsvParser goldCsvFile; if (!goldCsvFile.Open(csvPath.wstring(), testCase.processId)) { - Assert::Fail(L"Failed to open gold CSV file"); - return; + if (throwOnFailure) { + Assert::Fail(L"Failed to open gold CSV file"); + } + else { + throw std::runtime_error("Failed to open gold CSV file"); + } + return false; } std::unique_ptr pSession; if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { goldCsvFile.Close(); - Assert::Fail(L"Failed to setup test environment"); - return; + if (throwOnFailure) { + Assert::Fail(L"Failed to setup test environment"); + } + else { + throw std::runtime_error("Failed to setup test environment"); + } + return false; } // Track if we encountered an assertion failure @@ -290,50 +377,177 @@ namespace EtlTests if (testCase.isExpectedFailure) { if (testPassed) { // Test passed but was expected to fail - this is noteworthy! - Logger::WriteMessage(std::format( - "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" - " Expected failure reason: {}\n" - " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", - testCase.testName, testCase.failureReason).c_str()); - // Still pass the test since it's good news + if (throwOnFailure) { + Logger::WriteMessage(std::format( + "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", + testCase.testName, testCase.failureReason).c_str()); + } + // Return true because test technically passed (even though unexpected) + return true; } else { // Test failed as expected - Logger::WriteMessage(std::format( - "[FAIL] Expected failure: {}\n", - testCase.failureReason).c_str()); - // Re-throw to fail the test, but it's documented as expected - Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", - pmon::util::str::ToWide(testCase.failureReason)).c_str()); + if (throwOnFailure) { + // For individual test methods, log and assert + Logger::WriteMessage(std::format( + "[FAIL] Expected failure: {}\n", + testCase.failureReason).c_str()); + Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", + pmon::util::str::ToWide(testCase.failureReason)).c_str()); + } + // Return false to indicate test failed (even though expected) + return false; } } else if (!testPassed) { - // Unexpected failure - fail the test with detailed message - Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + // Unexpected failure + if (throwOnFailure) { + Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + } + else { + // For CSV runner, throw std::runtime_error so it can be caught and counted + throw std::runtime_error(exceptionMessage); + } + return false; } - // else: test passed and wasn't expected to fail - all good! + // Test passed and wasn't expected to fail - all good! + return true; } - void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) + // Run all test cases from a CSV file + void RunTestsFromCsv(const std::string& csvFilePath) { - auto testCase = FindTestCaseByName(caseName); - if (!testCase) { - Assert::Fail(L"Test case not found"); + // Load test cases from CSV + std::vector testCases; + try { + testCases = LoadTestCasesFromCsv(csvFilePath); + Logger::WriteMessage(std::format("Loaded {} test cases from {}\n", + testCases.size(), csvFilePath).c_str()); + } + catch (const std::exception& e) { + Assert::Fail(pmon::util::str::ToWide( + std::format("Failed to load test cases CSV: {}", e.what())).c_str()); return; } - fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" - : fs::path(GetAdditionalTestLocation().value_or("")); + // Statistics + int totalTests = 0; + int passedTests = 0; + int failedTests = 0; + int expectedFailures = 0; + int skippedTests = 0; + std::vector failureDetails; + + // Run each test case + for (const auto& testCase : testCases) { + // Skip if RunTest is false + if (!testCase.runTest) { + skippedTests++; + Logger::WriteMessage(std::format("[SKIP] {} (RunTest=false)\n", + testCase.testName).c_str()); + continue; + } + + totalTests++; + Logger::WriteMessage(std::format("\n=== Running Test {}/{}: {} ===\n", + totalTests, testCases.size() - skippedTests, testCase.testName).c_str()); + + // Determine test location + fs::path testPath = testCase.useAdditionalTestLocation + ? fs::path(GetAdditionalTestLocation().value_or("")) + : fs::path("..") / ".." / "tests" / "gold"; + + // Prepare debug CSV if requested + std::optional debugCsv; + if (testCase.produceDebugCsv) { + auto outputDir = testPath.string(); + auto debugCsvName = testCase.testName + "-debug"; + debugCsv = CreateCsvFile(outputDir, debugCsvName); + if (debugCsv.has_value()) { + Logger::WriteMessage(std::format(" Producing debug CSV: {}-debug.csv\n", + testCase.testName).c_str()); + } + } + + // Run the test + bool testPassed = false; + std::string errorMessage; + + try { + testPassed = RunGoldCsvTest(testCase, testPath.string(), debugCsv, false); // Returns true/false instead of throwing + } + catch (const std::exception& e) { + // Only unexpected failures throw exceptions + testPassed = false; + errorMessage = e.what(); + } + + // Handle the result + if (testPassed) { + if (testCase.isExpectedFailure) { + // Test passed but was expected to fail - this is noteworthy! + Logger::WriteMessage(std::format( + "[UNEXPECTED PASS] Test passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update CSV to set IsExpectedFailure = false\n", + testCase.failureReason).c_str()); + } + else { + Logger::WriteMessage(std::format("[PASS] {}\n", testCase.testName).c_str()); + } + passedTests++; + } + else { + // Test failed + if (testCase.isExpectedFailure) { + // Test failed as expected + expectedFailures++; + Logger::WriteMessage(std::format( + "[EXPECTED FAIL] {}\n Reason: {}\n", + testCase.testName, testCase.failureReason).c_str()); + } + else { + // Unexpected failure + failedTests++; + std::string detail = std::format("[FAIL] {}: {}", + testCase.testName, errorMessage.empty() ? "Test failed" : errorMessage); + failureDetails.push_back(detail); + Logger::WriteMessage(std::format("{}\n", detail).c_str()); + } + } - std::optional debugCsv; - if (createDebugCsv) { - auto outputDir = path.string(); - auto debugCsvName = testCase->processName + "-debug.csv"; - debugCsv = CreateCsvFile(outputDir, debugCsvName); + // Close debug CSV if it was created + if (debugCsv.has_value()) { + debugCsv->close(); + } } - RunGoldCsvTest(*testCase, path.string(), debugCsv); - if (debugCsv.has_value()) { - debugCsv->close(); + + // Print summary + Logger::WriteMessage(std::format( + "\n========================================\n" + "Test Summary\n" + "========================================\n" + "Total Test Cases in CSV: {}\n" + "Skipped (RunTest=false): {}\n" + "Tests Run: {}\n" + " Passed: {}\n" + " Failed (Unexpected): {}\n" + " Failed (Expected): {}\n" + "========================================\n", + testCases.size(), skippedTests, totalTests, + passedTests, failedTests, expectedFailures).c_str()); + + // Fail the overall test if there were unexpected failures + if (failedTests > 0) { + std::string summary = std::format( + "\n{} of {} tests failed unexpectedly:\n\n", + failedTests, totalTests); + for (const auto& detail : failureDetails) { + summary += detail + "\n"; + } + Assert::Fail(pmon::util::str::ToWide(summary).c_str()); } } public: @@ -356,6 +570,16 @@ namespace EtlTests goldCsvFile.Close(); } + // Example: Run all tests from CSV file + // This single test will run all test cases defined in the CSV + // Use the RunTest column in CSV to selectively enable/disable tests + TEST_METHOD(RunAllTestsFromCsv) + { + // CSV file is in the PresentMonAPI2Tests source directory + // Working dir is build/Debug, so go up to source tree + RunTestsFromCsv("..\\..\\IntelPresentMon\\PresentMonAPI2Tests\\test_cases.csv"); + } + TEST_METHOD(OpenServiceTest) { using namespace std::string_literals; @@ -383,174 +607,11 @@ namespace EtlTests } TEST_METHOD(OpenMockSessionTest) { - auto testCase = FindTestCaseByName("test_case_0_10792"); - auto etlFile = "..\\..\\tests\\gold\\" + testCase->etlFile; + // Simple test to verify we can create a session with an ETL file + const auto etlFile = "..\\..\\tests\\gold\\test_case_0.etl"; std::unique_ptr pSession; auto result = SetupTestEnvironment(etlFile, "10000", pSession); - Assert::IsTrue(result, L"SetupTestEnvironment failed"); - } - TEST_METHOD(Tc000v2Presenter10792) - { - RunTestFromCase("test_case_0_10792"); - } - TEST_METHOD(Tc000v2DWM1268) - { - RunTestFromCase("test_case_0_1268"); - } - - TEST_METHOD(Tc000v2Presenter8320) - { - RunTestFromCase("test_case_0_8320"); - } - TEST_METHOD(Tc000v2Presenter11648) - { - RunTestFromCase("test_case_0_11648"); - } - TEST_METHOD(Tc000v2Presenter3976) - { - RunTestFromCase("test_case_0_3976"); - } - TEST_METHOD(Tc000v2Presenter11112) - { - RunTestFromCase("test_case_0_11112"); - } - TEST_METHOD(Tc000v2Presenter2032) - { - RunTestFromCase("test_case_0_2032"); - } - TEST_METHOD(Tc000v2Presenter5988) - { - RunTestFromCase("test_case_0_5988"); - } - TEST_METHOD(Tc000v2Presenter12268) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_0_12268"); - } - TEST_METHOD(Tc000v2Presenter11100) - { - RunTestFromCase("test_case_0_11100"); - } - - TEST_METHOD(Tc001v2Dwm1564) - { - RunTestFromCase("test_case_1_1564", true, true); - } - TEST_METHOD(Tc001v2Presenter24560) - { - RunTestFromCase("test_case_1_24560"); - } - TEST_METHOD(Tc001v2devenv24944) - { - RunTestFromCase("test_case_1_24944"); - } - TEST_METHOD(Tc002v2Dwm1300) - { - RunTestFromCase("test_case_2_1300"); - } - TEST_METHOD(Tc002v2Presenter10016) - { - RunTestFromCase("test_case_2_10016"); - } - TEST_METHOD(Tc002v2Presenter5348) - { - RunTestFromCase("test_case_2_5348"); - } - TEST_METHOD(Tc002v2Presenter5220) - { - RunTestFromCase("test_case_2_5220"); - } - TEST_METHOD(Tc003v2Dwm1252) - { - RunTestFromCase("test_case_3_1252"); - } - TEST_METHOD(Tc003v2Presenter5892) - { - RunTestFromCase("test_case_3_5892"); - } - TEST_METHOD(Tc003v2Presenter10112) - { - RunTestFromCase("test_case_3_10112"); - } - TEST_METHOD(Tc003v2Presenter12980) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_3_12980"); - } - TEST_METHOD(Tc004v2Presenter5192) - { - RunTestFromCase("test_case_4_5192"); - } - TEST_METHOD(Tc004v2Presenter5236) - { - RunTestFromCase("test_case_4_5236"); - } - TEST_METHOD(Tc004v2Presenter8536) - { - RunTestFromCase("test_case_4_8536"); - } - TEST_METHOD(Tc004v2Presenter9620) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_4_9620"); - } - TEST_METHOD(Tc004v2Dwm10376) - { - RunTestFromCase("test_case_4_10376"); - } - TEST_METHOD(Tc005v2PresentBench24892) - { - RunTestFromCase("test_case_5_24892"); - } - TEST_METHOD(Tc006CP2077) - { - RunTestFromCase("test_case_6_10796", false); - } - TEST_METHOD(Tc007CP2077) - { - RunTestFromCase("test_case_7_11320", false); - } - TEST_METHOD(Tc008ACShadows) - { - RunTestFromCase("test_case_8_6920", false); - } - TEST_METHOD(Tc009F124) - { - RunTestFromCase("test_case_9_10340", false); - } - TEST_METHOD(Tc010NarakaBladepoint) - { - RunTestFromCase("test_case_10_9888", false); - } - TEST_METHOD(Tc011NarakaBladepoint) - { - RunTestFromCase("test_case_11_1524", false); - } - TEST_METHOD(Tc012F124) - { - RunTestFromCase("test_case_12_10168", false); - } - TEST_METHOD(Tc013Dingo) - { - RunTestFromCase("test_case_13_11780", false); + Assert::IsTrue(result, L"SetupTestEnvironment failed"); } }; } \ No newline at end of file From f3379980ebc2d1b4bdb22f172f2a8dd47de2dff0 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Mon, 10 Nov 2025 09:54:32 -0800 Subject: [PATCH 08/19] Adding in test cases csv --- .../PresentMonAPI2Tests/test_cases.csv | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 IntelPresentMon/PresentMonAPI2Tests/test_cases.csv diff --git a/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv new file mode 100644 index 00000000..7a74e9f3 --- /dev/null +++ b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv @@ -0,0 +1,36 @@ +TestName,ProcessID,ProcessName,EtlFile,GoldCsvFile,PollCount,WaitTimeSecs,IsExpectedFailure,FailureReason,UseAdditionalTestLocation,ProduceDebugCsv,RunTest +test_case_0_10792,10792,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_1268,1268,dwm.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_8320,8320,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11648,11648,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_3976,3976,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11112,11112,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_2032,2032,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_5988,5988,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_12268,12268,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11100,11100,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_1_1564,1564,dwm.exe,test_case_1.etl,test_case_1.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_1_24560,24560,Presenter.exe,test_case_1.etl,test_case_1.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_1_24944,24944,devenv.exe,test_case_1.etl,test_case_1.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_1300,1300,dwm.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_10016,10016,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_5348,5348,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_5220,5220,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_1252,1252,dwm.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_5892,5892,dwm.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_10112,10112,Presenter.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_12980,12980,Presenter.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_5192,5192,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_5236,5236,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_4_8536,8536,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_9620,9620,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_10376,10376,dwm.exe,test_case_4.etl,test_case_4.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_5_24892,24892,PresentBench.exe,test_case_5.etl,test_case_5.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_6_10796,10796,cpLauncher.exe,test_case_6.etl,test_case_6.csv,10,1,FALSE,,TRUE,FALSE,TRUE +test_case_7_11320,11320,cpLauncher.exe,test_case_7.etl,test_case_7.csv,20,5,FALSE,,TRUE,FALSE,TRUE +test_case_8_6920,6920,scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe,test_case_8.etl,test_case_8.csv,20,5,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_9_10340,10340,F1_24.exe,test_case_9.etl,test_case_9.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_10_9888,9888,NarakaBladepoint.exe,test_case_10.etl,test_case_10.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_11_1524,1524,NarakaBladepoint.exe,test_case_11.etl,test_case_11.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_12_10168,10168,F1_24.exe,test_case_12.etl,test_case_12.csv,20,5,FALSE,,TRUE,FALSE,TRUE +test_case_13_11780,11780,Dingo.Main_Win64_retail.exe,test_case_13.etl,test_case_13.csv,10,1,FALSE,,TRUE,FALSE,TRUE From fdba925ff33777590d7e1ae96501a9c7eb6921ad Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Tue, 4 Nov 2025 11:33:20 -0800 Subject: [PATCH 09/19] Streamlined ETL ULT tests Updated code to have a single test run path that all tests use to remove test setup and execution code found in every test. --- .../PresentMonAPI2Tests/EtlTests.cpp | 1860 ++--------------- 1 file changed, 205 insertions(+), 1655 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 4f854974..f66cac7f 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -9,15 +9,77 @@ #include "CsvHelper.h" #include "../PresentMonAPI2Loader/Loader.h" #include "../CommonUtilities/pipe/Pipe.h" +#include "../CommonUtilities/str/String.h" #include #include #include +#include using namespace Microsoft::VisualStudio::CppUnitTestFramework; namespace bp = boost::process::v1; +namespace fs = std::filesystem; namespace EtlTests { + static constexpr const char* controlPipe_ = R"(\\.\pipe\pm-etlults-ctrl)"; + static constexpr const char* introNsm_ = "pm_etlults_test_intro"; + static constexpr const char* nsmPrefix_ = "pmon_nsm_utest_"; + + // Necessary data for each test case + struct TestCaseData { + std::string testName; + uint32_t processId; + std::string processName; + std::string etlFile; + std::wstring goldCsvFile; + bool isExpectedFailure; + std::string failureReason; + }; + + static const TestCaseData GOLD_TEST_CASES[] { + {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, + {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, + {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, + {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_3_5192", 5192, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, + {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, + {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", false, ""}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", true, "Expected failure - SwapChain Initialization needed"}, + {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", false, ""}, + {"test_case_10_42132", 42132, "Marvel-Win64-Shipping.exe", "test_case_10.etl", L"test_case_10.csv", false, ""}, + {"test_case_11_12524", 12524, "Cyberpunk2077.exe", "test_case_11.etl", L"test_case_11.csv", false, ""}, + {"test_case_12_24412", 24412, "Marvel-Win64-Shipping.exe", "test_case_12.etl", L"test_case_12.csv", false, ""}, + }; + + TestCaseData* FindTestCaseByName(const std::string& testName) { + auto it = std::ranges::find_if(GOLD_TEST_CASES, + [&testName](const TestCaseData& tc) { return tc.testName == testName; }); + return it != std::end(GOLD_TEST_CASES) ? const_cast(&*it) : nullptr; + } + void RunTestCaseV2(std::unique_ptr&& pSession, const uint32_t& processId, const std::string& processName, CsvParser& goldCsvFile, std::optional& debugCsvFile) { @@ -95,6 +157,97 @@ namespace EtlTests TEST_CLASS(GoldEtlCsvTests) { std::optional oChild; + private: + std::optional GetAdditionalTestLocation() { + // Check for additional test directory from environment variable + // This allows developers to specify their own test directories without + // modifying the source code. Set PRESENTMON_ADDITIONAL_TEST_DIR environment + // variable or use a .runsettings.user file (see template). + std::wstring additionalTestDir; + wchar_t* envTestDir = nullptr; + size_t envTestDirLen = 0; + if (_wdupenv_s(&envTestDir, &envTestDirLen, L"PRESENTMON_ADDITIONAL_TEST_DIR") == 0 && envTestDir != nullptr) { + additionalTestDir = envTestDir; + free(envTestDir); + } + return additionalTestDir.empty() ? std::nullopt : std::make_optional(pmon::util::str::ToNarrow(additionalTestDir)); + } + + bool SetupTestEnvironment(const std::string& etlFile, const std::string& timedStop, std::unique_ptr& outSession) + { + using namespace std::string_literals; + using namespace std::chrono_literals; + + bp::ipstream out; + bp::opstream in; + + oChild.emplace("PresentMonService.exe"s, + "--timed-stop"s, timedStop, + "--control-pipe"s, controlPipe_, + "--nsm-prefix"s, nsmPrefix_, + "--intro-nsm"s, introNsm_, + "--etl-test-file"s, etlFile, + bp::std_out > out, bp::std_in < in); + + if (!pmon::util::pipe::DuplexPipe::WaitForAvailability(std::string(controlPipe_) + "-in", 500)) { + Assert::Fail(L"Timeout waiting for service control pipe"); + return false; + } + + try { + pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); + pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); + outSession = std::make_unique(controlPipe_); + return true; + } + catch (const std::exception& e) { + std::cout << "Error: " << e.what() << std::endl; + Assert::Fail(L"Failed to connect to service via named pipe"); + return false; + } + } + void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional debugCsv) + { + using namespace std::string_literals; + + if (testCase.isExpectedFailure) { + Assert::Fail(pmon::util::str::ToWide(testCase.failureReason).c_str()); + return; + } + + fs::path etlFile = fs::path(goldPath) / testCase.etlFile; + fs::path csvPath = fs::path(goldPath) / testCase.goldCsvFile; + + CsvParser goldCsvFile; + if (!goldCsvFile.Open(csvPath.wstring(), testCase.processId)) { + Assert::Fail(L"Failed to open gold CSV file"); + return; + } + + std::unique_ptr pSession; + if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { + goldCsvFile.Close(); + return; + } + + RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, + goldCsvFile, debugCsv); + + goldCsvFile.Close(); + } + void RunTestFromCase(const std::string& caseName, bool useDefault = true) + { + auto testCase = FindTestCaseByName(caseName); + if (!testCase) { + Assert::Fail(L"Test case not found"); + return; + } + + fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" + : fs::path(GetAdditionalTestLocation().value_or("")); + + RunGoldCsvTest(*testCase, path.string(), std::nullopt); + } public: TEST_METHOD_CLEANUP(Cleanup) { @@ -107,7 +260,6 @@ namespace EtlTests using namespace std::literals; std::this_thread::sleep_for(50ms); } - TEST_METHOD(OpenCsvTest) { const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; @@ -143,41 +295,11 @@ namespace EtlTests } TEST_METHOD(OpenMockSessionTest) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - + auto testCase = FindTestCaseByName("test_case_0_10792"); + auto etlFile = "..\\..\\tests\\gold\\" + testCase->etlFile; std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } + auto result = SetupTestEnvironment(etlFile, "10000", pSession); + Assert::IsTrue(result, L"SetupTestEnvironment failed"); } TEST_METHOD(ConsumeBlobsTest) @@ -253,395 +375,36 @@ namespace EtlTests } TEST_METHOD(Tc000v2Presenter10792) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10792; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_10792"); } TEST_METHOD(Tc000v2DWM1268) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1268; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_1268"); } + TEST_METHOD(Tc000v2Presenter8320) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 8320; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_8320"); } TEST_METHOD(Tc000v2Presenter11648) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11648; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11648"); } TEST_METHOD(Tc000v2Presenter3976) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 3976; - const std::string processName = "Presenter.exe"; - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - std::optional debugCsv; // Empty optional - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_3976"); } TEST_METHOD(Tc000v2Presenter11112) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11112; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11112"); } TEST_METHOD(Tc000v2Presenter2032) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 2032; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_2032"); } TEST_METHOD(Tc000v2Presenter5988) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5988; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_5988"); } TEST_METHOD(Tc000v2Presenter12268) { @@ -652,597 +415,52 @@ namespace EtlTests // the process is not active and exit. Need to add some type of synchronization // in mock presentmon session to not shutdown the session until notified // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12268; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_12268"); } TEST_METHOD(Tc000v2Presenter11100) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11100; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_0.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_0_11100"); } TEST_METHOD(Tc001v2Dwm1564) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // This test is an expected failure. The reason for the failure is the - // mock presentmon session is writing a present to the nsm that is - // earlier than the console application allows because of swap chain - // initialization that is not implemented in the mock presentmon session. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1564; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_1564"); } TEST_METHOD(Tc001v2Presenter24560) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24560; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_24560"); } TEST_METHOD(Tc001v2devenv24944) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24944; - const std::string processName = "devenv.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_1.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_1.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_1_24944"); } TEST_METHOD(Tc002v2Dwm1300) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1300; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_1300"); } TEST_METHOD(Tc002v2Presenter10016) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10016; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_10016"); } TEST_METHOD(Tc002v2Presenter5348) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5348; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_5348"); } TEST_METHOD(Tc002v2Presenter5220) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5220; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_2.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_2.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_2_5220"); } TEST_METHOD(Tc003v2Dwm1252) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 1252; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_1252"); } TEST_METHOD(Tc003v2Presenter5892) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5892; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_5892"); } TEST_METHOD(Tc003v2Presenter10112) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10112; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_10112"); } TEST_METHOD(Tc003v2Presenter12980) { @@ -1253,201 +471,19 @@ namespace EtlTests // the process is not active and exit. Need to add some type of synchronization // in mock presentmon session to not shutdown the session until notified // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12980; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_3.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_3.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_12980"); } - TEST_METHOD(Tc004v2Presenter5192) + TEST_METHOD(Tc003v2Presenter5192) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5192; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_3_5192"); } TEST_METHOD(Tc004v2Presenter5236) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 5236; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_4_5236"); } TEST_METHOD(Tc004v2Presenter8536) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 8536; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_4_8536"); } TEST_METHOD(Tc004v2Presenter9620) { @@ -1455,532 +491,46 @@ namespace EtlTests // finishedby the the mock presentmon session. If the ETL session finishes // and sets the process id to not active from the mock presentmon session // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 9620; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc004v2Dwm10376) - { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10376; - const std::string processName = "dwm.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_4.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_4.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc005v2PresentBench24892) - { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // Expected failure due to incorrect swap chain handling by middleware. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24892; - const std::string processName = "PresentBench.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_5.etl"; - const auto goldCsvName = L"..\\..\\tests\\gold\\test_case_5.csv"; - - CsvParser goldCsvFile; - goldCsvFile.Open(goldCsvName, processId); - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - } - TEST_METHOD(Tc006v2CPXellOn10796Ext) - { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10796; - const std::string processName = "cpLauncher.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_6.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_6.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + // the process is not active and exit. Need to add some type of synchronization + // in mock presentmon session to not shutdown the session until notified + // by close session call. + RunTestFromCase("test_case_4_9620"); + } + TEST_METHOD(Tc004v2Dwm10376) + { + RunTestFromCase("test_case_4_10376"); + } + TEST_METHOD(Tc005v2PresentBench24892) + { + RunTestFromCase("test_case_5_24892"); + } + TEST_METHOD(Tc006v2CPXellOn10796Ext) + { + RunTestFromCase("test_case_6_10796", false); } TEST_METHOD(Tc007v2CPXellOnFgOn11320Ext) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 11320; - const std::string processName = "cpLauncher.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_7.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_7.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_7_11320", false); } TEST_METHOD(Tc008v2ACSXellOnFgOn6920Ext) { - Assert::AreEqual(true, false, L"*** Expected Failure. WIP."); - // This test is an expected failure. The reason for the failure is the - // mock presentmon session is writing a present to the nsm that is - // earlier than the console application allows because of swap chain - // initialization that is not implemented in the mock presentmon session. - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 6920; - const std::string processName = "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_8.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_8.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); + RunTestFromCase("test_case_8_6920", false); } TEST_METHOD(Tc009v2F124XellOnFgOn10340Ext) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10340; - const std::string processName = "F1_24.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_9.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_9.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_9_10340", false); } TEST_METHOD(Tc010MarvelOnNvPcl1FgOnExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 42132; - const std::string processName = "Marvel-Win64-Shipping.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_10.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_10.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_10_42132", false); } TEST_METHOD(Tc011CP2077Pcl2FgOffRelexOffExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 12524; - const std::string processName = "Cyberpunk2077.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_11.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_11.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - oChild.emplace("PresentMonService.exe"s, - "--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - std::this_thread::sleep_for(1000ms); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_11_12524", false); } TEST_METHOD(Tc012MarvelOnNvPcl3FgOnAutoReflexOnFrameDelayExt) { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 24412; - const std::string processName = "Marvel-Win64-Shipping.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "F:\\EtlTesting\\test_case_12.etl"; - const auto goldCsvName = L"F:\\EtlTesting\\test_case_12.csv"; - - CsvParser goldCsvFile; - if (!goldCsvFile.Open(goldCsvName, processId)) { - return; - } - - std::string folder = "F:\\EtlTesting\\ETLDebugging\\testcase12\\"s; - std::string csvName = "debug.csv"s; - debugCsv = CreateCsvFile(folder,csvName); - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "60000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - std::this_thread::sleep_for(1000ms); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - RunTestCaseV2(std::move(pSession), processId, processName, goldCsvFile, debugCsv); - goldCsvFile.Close(); - if (debugCsv.has_value()) { - debugCsv->close(); - } + RunTestFromCase("test_case_12_24412", false); } }; } \ No newline at end of file From 9eef358dac082781f2262877559b444a202f4577 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Tue, 4 Nov 2025 11:35:25 -0800 Subject: [PATCH 10/19] Removed now redundant test ConsumeBlobs is no longer needed as every test case now uses the same path that was used by ConsumeBlobs --- .../PresentMonAPI2Tests/EtlTests.cpp | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index f66cac7f..fe275d62 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -301,78 +301,6 @@ namespace EtlTests auto result = SetupTestEnvironment(etlFile, "10000", pSession); Assert::IsTrue(result, L"SetupTestEnvironment failed"); } - - TEST_METHOD(ConsumeBlobsTest) - { - using namespace std::string_literals; - using namespace std::chrono_literals; - - const uint32_t processId = 10792; - const std::string processName = "Presenter.exe"; - std::optional debugCsv; // Empty optional - - bp::ipstream out; // Stream for reading the process's output - bp::opstream in; // Stream for writing to the process's input - - const auto pipeName = R"(\\.\pipe\test-pipe-pmsvc-2)"s; - const auto introName = "PM_intro_test_nsm_2"s; - const auto etlName = "..\\..\\tests\\gold\\test_case_0.etl"; - - oChild.emplace("PresentMonService.exe"s, - //"--timed-stop"s, "10000"s, - "--control-pipe"s, pipeName, - "--nsm-prefix"s, "pmon_nsm_utest_"s, - "--intro-nsm"s, introName, - "--etl-test-file"s, etlName, - bp::std_out > out, bp::std_in < in); - - Assert::IsTrue(pmon::util::pipe::DuplexPipe::WaitForAvailability(pipeName, 500), - L"Timeout waiting for service control pipe"); - - std::unique_ptr pSession; - { - try - { - pmLoaderSetPathToMiddlewareDll_("./PresentMonAPI2.dll"); - pmSetupODSLogging_(PM_DIAGNOSTIC_LEVEL_DEBUG, PM_DIAGNOSTIC_LEVEL_ERROR, false); - pSession = std::make_unique(pipeName); - } - catch (const std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - Assert::AreEqual(false, true, L"*** Connecting to service via named pipe"); - return; - } - } - - pmapi::ProcessTracker processTracker; - uint32_t totalFramesValidated = 0; - - PM_QUERY_ELEMENT queryElements[]{ - { PM_METRIC_BETWEEN_PRESENTS, PM_STAT_NONE, 0, 0}, - }; - - auto frameQuery = pSession->RegisterFrameQuery(queryElements); - auto blobs = frameQuery.MakeBlobContainer(8); - - processTracker = pSession->TrackProcess(processId); - - using Clock = std::chrono::high_resolution_clock; - const auto start = Clock::now(); - - while (1) { - frameQuery.Consume(processTracker, blobs); - if (blobs.GetNumBlobsPopulated() == 0) { - if (Clock::now() - start >= 1s) { - // if it takes longer than 1 second to consume the first frame, throw failure - Assert::Fail(L"Timeout waiting to consume first frame"); - } - std::this_thread::sleep_for(8ms); - } - else { - break; - } - } - } TEST_METHOD(Tc000v2Presenter10792) { RunTestFromCase("test_case_0_10792"); From 5304cf3b3bdcc5e27c78a77b76a18c88bb18adc6 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 10:38:27 -0800 Subject: [PATCH 11/19] Better name creation method and filesystem handling for Debug csvs --- .../PresentMonAPI2Tests/CsvHelper.h | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h index 6063da67..e6d6a2be 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h +++ b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h @@ -12,8 +12,10 @@ #include #include #include +#include using namespace Microsoft::VisualStudio::CppUnitTestFramework; +namespace fs = std::filesystem; enum Header { Header_Application, @@ -344,16 +346,31 @@ bool Validate(const T& param1, const T& param2) { } -std::optional CreateCsvFile(std::string& output_dir, std::string& processName) +std::optional CreateCsvFile(const std::string& output_dir, const std::string& processName) { // Setup csv file time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); tm local_time; localtime_s(&local_time, &now); - std::ofstream csvFile; - std::string csvFileName = output_dir + processName; + try { - csvFile.open(csvFileName); + // Use filesystem to construct the path properly + fs::path outputPath(output_dir); + fs::path fileName = processName + "_" + + std::to_string(local_time.tm_year + 1900) + + std::to_string(local_time.tm_mon + 1) + + std::to_string(local_time.tm_mday) + + std::to_string(local_time.tm_hour) + + std::to_string(local_time.tm_min) + + std::to_string(local_time.tm_sec) + ".csv"; + + fs::path fullPath = outputPath / fileName; + + std::ofstream csvFile(fullPath); + if (!csvFile.is_open()) { + return std::nullopt; + } + csvFile << "Application,ProcessID,SwapChainAddress,PresentRuntime" ",SyncInterval,PresentFlags,AllowsTearing,PresentMode" From 4e4bb48f04bddef3c5ec065725ae8162423f6be7 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 10:41:04 -0800 Subject: [PATCH 12/19] Fix for ULT ETL testing When testing ETLs through the ULT system at the start the circular buffer can be very small and cause the peek code to wrap around when looking for the next displayed frame. Use the PresentStartTime of the "next" displayed frame to ensure it is in fact after the current frame --- IntelPresentMon/Streamer/StreamClient.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/IntelPresentMon/Streamer/StreamClient.cpp b/IntelPresentMon/Streamer/StreamClient.cpp index 13dd765b..319fa661 100644 --- a/IntelPresentMon/Streamer/StreamClient.cpp +++ b/IntelPresentMon/Streamer/StreamClient.cpp @@ -389,6 +389,19 @@ PM_STATUS StreamClient::ConsumePtrToNextNsmFrameData(const PmNsmFrameData** pNsm *pNextFrame = nullptr; return PM_STATUS::PM_STATUS_SUCCESS; } + + if (pFrameDataOfNextDisplayed != nullptr) { + if ((*pNsmData)->present_event.PresentStartTime >= + (*pFrameDataOfNextDisplayed)->present_event.PresentStartTime) { + // The next displayed frame must be after the current frame. + // This is an error so reset the next_dequeue_idx back to where we first started. + next_dequeue_idx_ = previous_dequeue_idx; + // Also reset the current and next frame data pointers + *pNsmData = nullptr; + *pNextFrame = nullptr; + return PM_STATUS::PM_STATUS_SUCCESS; + } + } PeekPreviousFrames( pFrameDataOfLastPresented, pFrameDataOfLastAppPresented, From 44e1b8a00481261927f1e3e5bc412aa78fcb1c79 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Thu, 6 Nov 2025 12:23:12 -0800 Subject: [PATCH 13/19] Updated test cases Added pollCount and waitTimeSecs fields to allow for test cases that have longer lead in times to the targeted application's first frame. Also fixed incorrect naming of various test cases. --- .../PresentMonAPI2Tests/EtlTests.cpp | 126 ++++++++++-------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index fe275d62..4dc911e5 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -32,46 +32,49 @@ namespace EtlTests std::string processName; std::string etlFile; std::wstring goldCsvFile; + int pollCount; + int waitTimeSecs; bool isExpectedFailure; std::string failureReason; }; static const TestCaseData GOLD_TEST_CASES[] { - {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, - {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", false, ""}, - {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", false, ""}, - {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_3_5192", 5192, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", false, ""}, - {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", false, ""}, - {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", true, "Expected failure - SwapChain Initialization needed"}, - {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", false, ""}, - {"test_case_10_42132", 42132, "Marvel-Win64-Shipping.exe", "test_case_10.etl", L"test_case_10.csv", false, ""}, - {"test_case_11_12524", 12524, "Cyberpunk2077.exe", "test_case_11.etl", L"test_case_11.csv", false, ""}, - {"test_case_12_24412", 24412, "Marvel-Win64-Shipping.exe", "test_case_12.etl", L"test_case_12.csv", false, ""}, + {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, + {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, + {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, + {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, + {"test_case_4_5192", 5192, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, + {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 10, 1, false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_12_10168", 10168, "F1_24.exe", "test_case_12.etl", L"test_case_12.csv", 20, 5, false, ""}, + {"test_case_13_11780", 11780, "Dingo.Main_Win64_retail.exe", "test_case_13.etl", L"test_case_13.csv", 10, 1, false, ""}, }; TestCaseData* FindTestCaseByName(const std::string& testName) { @@ -82,10 +85,10 @@ namespace EtlTests void RunTestCaseV2(std::unique_ptr&& pSession, const uint32_t& processId, const std::string& processName, CsvParser& goldCsvFile, - std::optional& debugCsvFile) { + std::optional& debugCsvFile, int pollCount, int waitTimeSecs) { using namespace std::chrono_literals; pmapi::ProcessTracker processTracker; - static constexpr uint32_t numberOfBlobs = 10000u; + static constexpr uint32_t numberOfBlobs = 2000; uint32_t totalFramesValidated = 0; PM_QUERY_ELEMENT queryElements[]{ @@ -134,12 +137,12 @@ namespace EtlTests frameQuery.Consume(processTracker, blobs); if (blobs.GetNumBlobsPopulated() == 0) { // if we poll 10 times in a row and get no new frames, consider this ETL finished - if (++emptyPollCount >= 10) { + if (++emptyPollCount >= pollCount) { if (totalFramesValidated > 0) { // only finish if we have consumed at least one frame break; } - else if (Clock::now() - start >= 1s) { + else if (Clock::now() - start >= std::chrono::seconds(waitTimeSecs)) { // if it takes longer than 1 second to consume the first frame, throw failure Assert::Fail(L"Timeout waiting to consume first frame"); } @@ -206,7 +209,7 @@ namespace EtlTests return false; } } - void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional debugCsv) + void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv) { using namespace std::string_literals; @@ -231,11 +234,11 @@ namespace EtlTests } RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, - goldCsvFile, debugCsv); + goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); goldCsvFile.Close(); } - void RunTestFromCase(const std::string& caseName, bool useDefault = true) + void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) { auto testCase = FindTestCaseByName(caseName); if (!testCase) { @@ -246,7 +249,16 @@ namespace EtlTests fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" : fs::path(GetAdditionalTestLocation().value_or("")); - RunGoldCsvTest(*testCase, path.string(), std::nullopt); + std::optional debugCsv; + if (createDebugCsv) { + auto outputDir = path.string(); + auto debugCsvName = testCase->processName + "-debug.csv"; + debugCsv = CreateCsvFile(outputDir, debugCsvName); + } + RunGoldCsvTest(*testCase, path.string(), debugCsv); + if (debugCsv.has_value()) { + debugCsv->close(); + } } public: TEST_METHOD_CLEANUP(Cleanup) @@ -401,9 +413,9 @@ namespace EtlTests // by close session call. RunTestFromCase("test_case_3_12980"); } - TEST_METHOD(Tc003v2Presenter5192) + TEST_METHOD(Tc004v2Presenter5192) { - RunTestFromCase("test_case_3_5192"); + RunTestFromCase("test_case_4_5192", true, true); } TEST_METHOD(Tc004v2Presenter5236) { @@ -432,33 +444,37 @@ namespace EtlTests { RunTestFromCase("test_case_5_24892"); } - TEST_METHOD(Tc006v2CPXellOn10796Ext) + TEST_METHOD(Tc006CP2077) { RunTestFromCase("test_case_6_10796", false); } - TEST_METHOD(Tc007v2CPXellOnFgOn11320Ext) + TEST_METHOD(Tc007CP2077) { RunTestFromCase("test_case_7_11320", false); } - TEST_METHOD(Tc008v2ACSXellOnFgOn6920Ext) + TEST_METHOD(Tc008ACShadows) { RunTestFromCase("test_case_8_6920", false); } - TEST_METHOD(Tc009v2F124XellOnFgOn10340Ext) + TEST_METHOD(Tc009F124) { RunTestFromCase("test_case_9_10340", false); } - TEST_METHOD(Tc010MarvelOnNvPcl1FgOnExt) + TEST_METHOD(Tc010NarakaBladepoint) + { + RunTestFromCase("test_case_10_9888", false); + } + TEST_METHOD(Tc011NarakaBladepoint) { - RunTestFromCase("test_case_10_42132", false); + RunTestFromCase("test_case_11_1524", false); } - TEST_METHOD(Tc011CP2077Pcl2FgOffRelexOffExt) + TEST_METHOD(Tc012F124) { - RunTestFromCase("test_case_11_12524", false); + RunTestFromCase("test_case_12_10168", false); } - TEST_METHOD(Tc012MarvelOnNvPcl3FgOnAutoReflexOnFrameDelayExt) + TEST_METHOD(Tc013Dingo) { - RunTestFromCase("test_case_12_24412", false); + RunTestFromCase("test_case_13_11780", false); } }; } \ No newline at end of file From b12ad58ff9ff1417536d70e7d1768fa7ac1d4544 Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Fri, 7 Nov 2025 09:29:37 -0800 Subject: [PATCH 14/19] Updated CsvHelper to use exceptions instead of MSTest Assert Provides better logging when failures occur. --- .../PresentMonAPI2Tests/CsvHelper.h | 302 ++++++++---------- .../PresentMonAPI2Tests/EtlTests.cpp | 104 +++++- 2 files changed, 229 insertions(+), 177 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h index e6d6a2be..bdbb2632 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h +++ b/IntelPresentMon/PresentMonAPI2Tests/CsvHelper.h @@ -178,6 +178,63 @@ std::wstring CreateErrorString(Header columnId, size_t line) return errorMessage; } +class CsvException : public std::runtime_error { +public: + explicit CsvException(const std::string& msg) : std::runtime_error(msg) {} +}; + +class CsvConversionException : public CsvException { + Header columnId_; + size_t line_; + std::string value_; +public: + CsvConversionException(Header columnId, size_t line, const std::string& value) + : CsvException(std::format("Invalid {} at line {}: '{}'", + GetHeaderString(columnId), line, value)) + , columnId_(columnId) + , line_(line) + , value_(value) + { + } + + Header GetColumnId() const { return columnId_; } + size_t GetLine() const { return line_; } + const std::string& GetValue() const { return value_; } +}; + +class CsvFileException : public CsvException { +public: + explicit CsvFileException(const std::string& msg) : CsvException(msg) {} +}; + +class CsvValidationException : public CsvException { + Header columnId_; + size_t line_; + + // Helper to format values for error messages + template + static auto FormatValue(const T& value) { + if constexpr (std::is_enum_v) { + return static_cast>(value); + } else { + return value; + } + } + +public: + template + CsvValidationException(Header columnId, size_t line, const T& receivedValue, const T& expectedValue) + : CsvException(std::format("{} at line {}: Do not match. Received value {}, Expected value {}", + GetHeaderString(columnId), line, FormatValue(receivedValue), FormatValue(expectedValue))) + , columnId_(columnId) + , line_(line) + { + } + + Header GetColumnId() const { return columnId_; } + size_t GetLine() const { return line_; } +}; + template class CharConvert { public: @@ -192,7 +249,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stod(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -201,7 +258,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stoul(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -210,7 +267,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = std::stoi(data); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } else if constexpr (std::is_same::value) { @@ -230,7 +287,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = static_cast(result1); } catch (...) { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } @@ -245,7 +302,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_GRAPHICS_RUNTIME_UNKNOWN; } else { - Assert::Fail(CreateErrorString(columnId, line).c_str()); + throw CsvConversionException(columnId, line, data); } } @@ -275,7 +332,7 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_PRESENT_MODE_UNKNOWN; } else { - Assert::Fail(CreateErrorString(Header_PresentMode, line).c_str()); + throw CsvConversionException(Header_PresentMode, line, data); } } else if constexpr (std::is_same::value) { @@ -298,12 +355,12 @@ void CharConvert::Convert(const std::string data, T& convertedData, Header co convertedData = PM_FRAME_TYPE_INTEL_XEFG; } else { - Assert::Fail(CreateErrorString(Header_FrameType, line).c_str()); + throw CsvConversionException(Header_FrameType, line, data); } } else { - Assert::Fail(CreateErrorString(UnknownHeader, line).c_str()); + throw CsvException("Unknown type conversion requested"); } } @@ -326,6 +383,44 @@ size_t countDecimalPlaces(double value) { return count; } +template +void ValidateOrThrow(Header columnId, size_t line, const T& received, const T& expected) { + if constexpr (std::is_same_v) { + // Both NaN is considered a match (both represent missing/invalid data) + if (std::isnan(received) && std::isnan(expected)) { + return; + } + // One NaN and one value is a mismatch + if (std::isnan(received) || std::isnan(expected)) { + throw CsvValidationException(columnId, line, received, expected); + } + // Both are valid numbers, compare with threshold + auto min = std::min(countDecimalPlaces(received), countDecimalPlaces(expected)); + double threshold = pow(0.1, min); + double difference = received - expected; + if (difference <= -threshold || difference >= threshold) { + throw CsvValidationException(columnId, line, received, expected); + } + } + else { + if (received != expected) { + throw CsvValidationException(columnId, line, received, expected); + } + } +} + +// Overload for std::optional - treats empty optional as NaN +void ValidateOrThrow(Header columnId, size_t line, const std::optional& received, double expected) { + if (received.has_value()) { + ValidateOrThrow(columnId, line, received.value(), expected); + } else { + // No value in CSV (optional is empty), expected should be NaN + if (!std::isnan(expected)) { + throw CsvValidationException(columnId, line, std::nan(""), expected); + } + } +} + template bool Validate(const T& param1, const T& param2) { if constexpr (std::is_same::value) { @@ -605,209 +700,93 @@ bool CsvParser::VerifyBlobAgainstCsv(const std::string& processName, const unsig // Go through all of the active headers and validate results for (const auto& pair : activeColHeadersMap_) { - - bool columnsMatch = false; switch (pair.second) { case Header_Application: - columnsMatch = true; + // Skip validation for application name break; case Header_ProcessID: - columnsMatch = Validate(v2MetricRow_.processId, processId_); + ValidateOrThrow(pair.second, line_, v2MetricRow_.processId, processId_); break; case Header_SwapChainAddress: - columnsMatch = Validate(v2MetricRow_.swapChain, swapChain); + ValidateOrThrow(pair.second, line_, v2MetricRow_.swapChain, swapChain); break; case Header_Runtime: - columnsMatch = Validate(v2MetricRow_.runtime, graphicsRuntime); + ValidateOrThrow(pair.second, line_, v2MetricRow_.runtime, graphicsRuntime); break; case Header_SyncInterval: - columnsMatch = Validate(v2MetricRow_.syncInterval, syncInterval); + ValidateOrThrow(pair.second, line_, v2MetricRow_.syncInterval, syncInterval); break; case Header_PresentFlags: - columnsMatch = Validate(v2MetricRow_.presentFlags, presentFlags); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentFlags, presentFlags); break; case Header_AllowsTearing: - columnsMatch = Validate(v2MetricRow_.allowsTearing, (uint32_t)allowsTearing); + ValidateOrThrow(pair.second, line_, v2MetricRow_.allowsTearing, (uint32_t)allowsTearing); break; case Header_PresentMode: - columnsMatch = Validate(v2MetricRow_.presentMode, presentMode); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentMode, presentMode); break; case Header_FrameType: - columnsMatch = Validate(v2MetricRow_.frameType, frameType); + ValidateOrThrow(pair.second, line_, v2MetricRow_.frameType, frameType); break; case Header_TimeInSeconds: - columnsMatch = Validate(v2MetricRow_.presentStartQPC, timeQpc); + ValidateOrThrow(pair.second, line_, v2MetricRow_.presentStartQPC, timeQpc); break; case Header_CPUStartQPC: - columnsMatch = Validate(v2MetricRow_.cpuFrameQpc, cpuStartQpc); + ValidateOrThrow(pair.second, line_, v2MetricRow_.cpuFrameQpc, cpuStartQpc); break; case Header_MsBetweenAppStart: - columnsMatch = Validate(v2MetricRow_.msBetweenAppStart, msBetweenAppStart); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenAppStart, msBetweenAppStart); break; case Header_MsCPUBusy: - columnsMatch = Validate(v2MetricRow_.msCpuBusy, msCpuBusy); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msCpuBusy, msCpuBusy); break; case Header_MsCPUWait: - columnsMatch = Validate(v2MetricRow_.msCpuWait, msCpuWait); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msCpuWait, msCpuWait); break; case Header_MsGPULatency: - columnsMatch = Validate(v2MetricRow_.msGpuLatency, msGpuLatency); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuLatency, msGpuLatency); break; case Header_MsGPUTime: - columnsMatch = Validate(v2MetricRow_.msGpuTime, msGpuTime); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuTime, msGpuTime); break; case Header_MsGPUBusy: - columnsMatch = Validate(v2MetricRow_.msGpuBusy, msGpuBusy); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuBusy, msGpuBusy); break; case Header_MsGPUWait: - columnsMatch = Validate(v2MetricRow_.msGpuWait, msGpuWait); + ValidateOrThrow(pair.second, line_, v2MetricRow_.msGpuWait, msGpuWait); break; case Header_MsBetweenSimulationStart: - if (v2MetricRow_.msBetweenSimStart.has_value()) { - columnsMatch = Validate(v2MetricRow_.msBetweenSimStart.value(), msBetweenSimStartTime); - } - else - { - if (std::isnan(msBetweenSimStartTime)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenSimStart, msBetweenSimStartTime); break; case Header_MsUntilDisplayed: - if (v2MetricRow_.msUntilDisplayed.has_value()) { - columnsMatch = Validate(v2MetricRow_.msUntilDisplayed.value(), msUntilDisplayed); - } - else - { - if (std::isnan(msUntilDisplayed)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msUntilDisplayed, msUntilDisplayed); break; case Header_MsBetweenDisplayChange: - if (v2MetricRow_.msBetweenDisplayChange.has_value()) { - columnsMatch = Validate(v2MetricRow_.msBetweenDisplayChange.value(), msBetweenDisplayChange); - } - else - { - if (std::isnan(msBetweenDisplayChange)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msBetweenDisplayChange, msBetweenDisplayChange); break; case Header_MsPCLatency: - if (v2MetricRow_.msPcLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msPcLatency.value(), msPcLatency); - } - else - { - if (std::isnan(msPcLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msPcLatency, msPcLatency); break; case Header_MsAnimationError: - if (v2MetricRow_.msAnimationError.has_value()) { - columnsMatch = Validate(v2MetricRow_.msAnimationError.value(), msAnimationError); - } - else - { - if (std::isnan(msAnimationError)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msAnimationError, msAnimationError); break; case Header_AnimationTime: - if (v2MetricRow_.animationTime.has_value()) { - columnsMatch = Validate(v2MetricRow_.animationTime.value(), animationTime); - } - else - { - if (std::isnan(animationTime)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.animationTime, animationTime); break; case Header_MsClickToPhotonLatency: - if (v2MetricRow_.msClickToPhotonLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msClickToPhotonLatency.value(), msClickToPhotonLatency); - } - else - { - if (std::isnan(msClickToPhotonLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msClickToPhotonLatency, msClickToPhotonLatency); break; case Header_MsAllInputToPhotonLatency: - if (v2MetricRow_.msAllInputToPhotonLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msAllInputToPhotonLatency.value(), msAllInputToPhotonLatency); - } - else - { - if (std::isnan(msAllInputToPhotonLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msAllInputToPhotonLatency, msAllInputToPhotonLatency); break; case Header_MsInstrumentedLatency: - if (v2MetricRow_.msInstrumentedLatency.has_value()) { - columnsMatch = Validate(v2MetricRow_.msInstrumentedLatency.value(), msInstrumentedLatency); - } - else - { - if (std::isnan(msInstrumentedLatency)) { - columnsMatch = true; - } - else - { - columnsMatch = false; - } - } + ValidateOrThrow(pair.second, line_, v2MetricRow_.msInstrumentedLatency, msInstrumentedLatency); break; default: - columnsMatch = true; + // Unknown header, skip validation break; } - if (columnsMatch == false) { - // If the columns do not match, create an error string - // and assert failure. - Assert::Fail(CreateErrorString(pair.second, line_).c_str()); - } - Assert::IsTrue(columnsMatch, CreateErrorString(pair.second, line_).c_str()); } } @@ -854,7 +833,8 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { cols_.clear(); if (_wfopen_s(&fp_, path.c_str(), L"r")) { - return false; + throw CsvFileException(std::format("Failed to open CSV file: {}", + pmon::util::str::ToNarrow(path))); } // Remove UTF-8 marker if there is one. @@ -880,9 +860,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { default: if ((size_t)h < KnownHeaderCount) { if (headerColumnIndex_[(size_t)h] != SIZE_MAX) { - std::wstring errorMessage = L"Duplicate column: "; - errorMessage += pmon::util::str::ToWide(cols_[i]); - Assert::Fail(errorMessage.c_str()); + throw CsvFileException(std::format("Duplication column: {}", cols_[i])); } else { headerColumnIndex_[(size_t)h] = i; @@ -890,8 +868,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { break; } else { - std::wstring errorMessage = L"Index outside of known headers."; - Assert::Fail(errorMessage.c_str()); + throw CsvFileException("Index outside of known headers"); } } } @@ -925,7 +902,7 @@ bool CsvParser::Open(std::wstring const& path, uint32_t processId) { Header_MsPCLatency}); if (!columnsOK) { - Assert::Fail(L"Missing required columns"); + throw CsvFileException("Missing required columns in CSV file"); } // Create a vector of active headers to be used when reading @@ -1213,7 +1190,7 @@ void CsvParser::ConvertToMetricDataType(const char* data, Header columnId) } break; default: - Assert::Fail(CreateErrorString(UnknownHeader, line_).c_str()); + throw CsvConversionException(UnknownHeader, line_, data); } return; @@ -1227,9 +1204,8 @@ bool CsvParser::ReadRow(bool gatherMetrics) // Read a line if (fgets(row_, _countof(row_), fp_) == nullptr) { if (ferror(fp_) != 0) { - std::wstring errorMessage = L"File read error at line: "; - errorMessage += std::to_wstring(line_); - Assert::Fail(errorMessage.c_str()); + throw CsvFileException(std::format("File read error at line: {}", + std::to_string(line_))); } return false; } diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 4dc911e5..97293fdb 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2022-2023 Intel Corporation +// Copyright (C) 2022-2023 Intel Corporation // SPDX-License-Identifier: MIT #include "../CommonUtilities/win/WinAPI.h" #include @@ -49,7 +49,7 @@ namespace EtlTests {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, @@ -68,8 +68,8 @@ namespace EtlTests {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 10, 1, false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, + {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 20, 5, false, ""}, + {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 20, 5, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, @@ -213,12 +213,7 @@ namespace EtlTests { using namespace std::string_literals; - if (testCase.isExpectedFailure) { - Assert::Fail(pmon::util::str::ToWide(testCase.failureReason).c_str()); - return; - } - - fs::path etlFile = fs::path(goldPath) / testCase.etlFile; + fs::path etlFile = fs::path(goldPath) / testCase.etlFile; fs::path csvPath = fs::path(goldPath) / testCase.goldCsvFile; CsvParser goldCsvFile; @@ -230,14 +225,95 @@ namespace EtlTests std::unique_ptr pSession; if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { goldCsvFile.Close(); + Assert::Fail(L"Failed to setup test environment"); return; } - RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, - goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); + // Track if we encountered an assertion failure + bool testPassed = true; + std::string exceptionMessage; + + try { + RunTestCaseV2(std::move(pSession), testCase.processId, testCase.processName, + goldCsvFile, debugCsv, testCase.pollCount, testCase.waitTimeSecs); + } + catch (const CsvValidationException& e) { + testPassed = false; + exceptionMessage = std::format( + "CSV Validation Error:\n" + " Column: {}\n" + " Line: {}\n" + " Details: {}", + GetHeaderString(e.GetColumnId()), + e.GetLine(), + e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvConversionException& e) { + testPassed = false; + exceptionMessage = std::format( + "CSV Conversion Error:\n" + " Column: {}\n" + " Line: {}\n" + " Invalid Value: '{}'\n" + " Details: {}", + GetHeaderString(e.GetColumnId()), + e.GetLine(), + e.GetValue(), + e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvFileException& e) { + testPassed = false; + exceptionMessage = std::format("CSV File Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const CsvException& e) { + testPassed = false; + exceptionMessage = std::format("CSV Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (const std::exception& e) { + testPassed = false; + exceptionMessage = std::format("Unexpected Error: {}", e.what()); + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } + catch (...) { + testPassed = false; + exceptionMessage = "Unknown exception caught"; + Logger::WriteMessage(std::format("[ERROR] {}\n", exceptionMessage).c_str()); + } goldCsvFile.Close(); + + // Now handle expected failure logic + if (testCase.isExpectedFailure) { + if (testPassed) { + // Test passed but was expected to fail - this is noteworthy! + Logger::WriteMessage(std::format( + "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", + testCase.testName, testCase.failureReason).c_str()); + // Still pass the test since it's good news + } + else { + // Test failed as expected + Logger::WriteMessage(std::format( + "[FAIL] Expected failure: {}\n", + testCase.failureReason).c_str()); + // Re-throw to fail the test, but it's documented as expected + Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", + pmon::util::str::ToWide(testCase.failureReason)).c_str()); + } + } + else if (!testPassed) { + // Unexpected failure - fail the test with detailed message + Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + } + // else: test passed and wasn't expected to fail - all good! } + void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) { auto testCase = FindTestCaseByName(caseName); @@ -364,7 +440,7 @@ namespace EtlTests TEST_METHOD(Tc001v2Dwm1564) { - RunTestFromCase("test_case_1_1564"); + RunTestFromCase("test_case_1_1564", true, true); } TEST_METHOD(Tc001v2Presenter24560) { @@ -415,7 +491,7 @@ namespace EtlTests } TEST_METHOD(Tc004v2Presenter5192) { - RunTestFromCase("test_case_4_5192", true, true); + RunTestFromCase("test_case_4_5192"); } TEST_METHOD(Tc004v2Presenter5236) { From c4ae1253949e021f4c18a01247a3ea589c156bea Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Mon, 10 Nov 2025 09:54:10 -0800 Subject: [PATCH 15/19] Removed single test run support for IPM ETL ULTs --- .../PresentMonAPI2Tests/EtlTests.cpp | 551 ++++++++++-------- 1 file changed, 306 insertions(+), 245 deletions(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index 97293fdb..db8f0fe0 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -25,7 +25,7 @@ namespace EtlTests static constexpr const char* introNsm_ = "pm_etlults_test_intro"; static constexpr const char* nsmPrefix_ = "pmon_nsm_utest_"; - // Necessary data for each test case + // Test case data structure - loaded from CSV file struct TestCaseData { std::string testName; uint32_t processId; @@ -36,51 +36,125 @@ namespace EtlTests int waitTimeSecs; bool isExpectedFailure; std::string failureReason; + bool useAdditionalTestLocation; // Load from additional test directory (runsettings) + bool produceDebugCsv; // Generate debug CSV output + bool runTest; // Whether to run this test (for selective debugging) }; - static const TestCaseData GOLD_TEST_CASES[] { - {"test_case_0_10792", 10792, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_1268", 1268, "dwm.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_8320", 8320, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11648", 11648, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_3976", 3976, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11112", 11112, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_2032", 2032, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_5988", 5988, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_12268", 12268, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_0_11100", 11100, "Presenter.exe", "test_case_0.etl", L"test_case_0.csv", 10, 1, false, ""}, - {"test_case_1_1564", 1564, "dwm.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_1_24560", 24560, "Presenter.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_1_24944", 24944, "devenv.exe", "test_case_1.etl", L"test_case_1.csv", 10, 1, false, ""}, - {"test_case_2_1300", 1300, "dwm.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_10016", 10016, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_5348", 5348, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_2_5220", 5220, "Presenter.exe", "test_case_2.etl", L"test_case_2.csv", 10, 1, false, ""}, - {"test_case_3_1252", 1252, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_5892", 5892, "dwm.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_10112", 10112, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_3_12980", 12980, "Presenter.exe", "test_case_3.etl", L"test_case_3.csv", 10, 1, false, ""}, - {"test_case_4_5192", 5192, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_12980", 12980, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_5236", 5236, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_4_8536", 8536, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_9620", 9620, "Presenter.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, false, ""}, - {"test_case_4_10376", 10376, "dwm.exe", "test_case_4.etl", L"test_case_4.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_5_24892", 24892, "PresentBench.exe", "test_case_5.etl", L"test_case_5.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_6_10796", 10796, "cpLauncher.exe", "test_case_6.etl", L"test_case_6.csv", 10, 1, false, ""}, - {"test_case_7_11320", 11320, "cpLauncher.exe", "test_case_7.etl", L"test_case_7.csv", 20, 5, false, ""}, - {"test_case_8_6920", 6920, "scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe", "test_case_8.etl", L"test_case_8.csv", 20, 5, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_9_10340", 10340, "F1_24.exe", "test_case_9.etl", L"test_case_9.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_10_9888", 9888, "NarakaBladepoint.exe", "test_case_10.etl", L"test_case_10.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_11_1524", 1524, "NarakaBladepoint.exe", "test_case_11.etl", L"test_case_11.csv", 10, 1, true, "Expected failure - Multiple SwapChain support needed"}, - {"test_case_12_10168", 10168, "F1_24.exe", "test_case_12.etl", L"test_case_12.csv", 20, 5, false, ""}, - {"test_case_13_11780", 11780, "Dingo.Main_Win64_retail.exe", "test_case_13.etl", L"test_case_13.csv", 10, 1, false, ""}, - }; + // Helper function to parse boolean from CSV string + bool ParseBool(const std::string& value) { + std::string lower = value; + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + return lower == "true" || lower == "1" || lower == "yes" || lower == "y"; + } - TestCaseData* FindTestCaseByName(const std::string& testName) { - auto it = std::ranges::find_if(GOLD_TEST_CASES, - [&testName](const TestCaseData& tc) { return tc.testName == testName; }); - return it != std::end(GOLD_TEST_CASES) ? const_cast(&*it) : nullptr; + // Helper function to trim whitespace from string + std::string Trim(const std::string& str) { + const auto start = str.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return ""; + const auto end = str.find_last_not_of(" \t\r\n"); + return str.substr(start, end - start + 1); + } + + // Parse a CSV line handling quoted fields + std::vector ParseCsvLine(const std::string& line) { + std::vector fields; + std::string field; + bool inQuotes = false; + + for (size_t i = 0; i < line.length(); ++i) { + char c = line[i]; + + if (c == '"') { + inQuotes = !inQuotes; + } + else if (c == ',' && !inQuotes) { + fields.push_back(Trim(field)); + field.clear(); + } + else { + field += c; + } + } + fields.push_back(Trim(field)); + return fields; + } + + // Load test cases from CSV file + // CSV Format: TestName,ProcessID,ProcessName,EtlFile,GoldCsvFile,PollCount,WaitTimeSecs,IsExpectedFailure,FailureReason,UseAdditionalTestLocation,ProduceDebugCsv,RunTest + std::vector LoadTestCasesFromCsv(const std::string& csvFilePath) { + std::vector testCases; + + // Convert to absolute path for better error reporting + fs::path absolutePath = fs::absolute(csvFilePath); + + std::ifstream file(csvFilePath); + if (!file.is_open()) { + throw std::runtime_error(std::format( + "Failed to open test cases CSV file:\n" + " Requested path: {}\n" + " Absolute path: {}\n" + " Current directory: {}", + csvFilePath, + absolutePath.string(), + fs::current_path().string())); + } + + std::string line; + bool isFirstLine = true; + size_t lineNumber = 0; + + while (std::getline(file, line)) { + lineNumber++; + + // Skip empty lines + if (Trim(line).empty()) { + continue; + } + + // Skip header line + if (isFirstLine) { + isFirstLine = false; + continue; + } + + try { + auto fields = ParseCsvLine(line); + + // Validate field count + if (fields.size() < 12) { + throw std::runtime_error(std::format( + "Line {}: Expected at least 12 fields, got {}", + lineNumber, fields.size())); + } + + TestCaseData testCase; + testCase.testName = fields[0]; + testCase.processId = std::stoul(fields[1]); + testCase.processName = fields[2]; + testCase.etlFile = fields[3]; + testCase.goldCsvFile = pmon::util::str::ToWide(fields[4]); + testCase.pollCount = std::stoi(fields[5]); + testCase.waitTimeSecs = std::stoi(fields[6]); + testCase.isExpectedFailure = ParseBool(fields[7]); + testCase.failureReason = fields[8]; + testCase.useAdditionalTestLocation = ParseBool(fields[9]); + testCase.produceDebugCsv = ParseBool(fields[10]); + testCase.runTest = ParseBool(fields[11]); + + testCases.push_back(testCase); + } + catch (const std::exception& e) { + throw std::runtime_error(std::format( + "Error parsing line {}: {}", lineNumber, e.what())); + } + } + + if (testCases.empty()) { + throw std::runtime_error("No test cases loaded from CSV file"); + } + + return testCases; } void RunTestCaseV2(std::unique_ptr&& pSession, @@ -143,8 +217,8 @@ namespace EtlTests break; } else if (Clock::now() - start >= std::chrono::seconds(waitTimeSecs)) { - // if it takes longer than 1 second to consume the first frame, throw failure - Assert::Fail(L"Timeout waiting to consume first frame"); + // if it takes longer than alloted test time to consume the first frame, throw failure + throw CsvException("Timeout waiting to consume first frame"); } } std::this_thread::sleep_for(8ms); @@ -209,7 +283,10 @@ namespace EtlTests return false; } } - void RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv) + // Returns true if test passed, false if test failed + // When throwOnFailure=false, returns status instead of throwing + // When throwOnFailure=true, throws Assert::Fail on failure + bool RunGoldCsvTest(const TestCaseData& testCase, const std::string& goldPath, std::optional& debugCsv, bool throwOnFailure = true) { using namespace std::string_literals; @@ -218,15 +295,25 @@ namespace EtlTests CsvParser goldCsvFile; if (!goldCsvFile.Open(csvPath.wstring(), testCase.processId)) { - Assert::Fail(L"Failed to open gold CSV file"); - return; + if (throwOnFailure) { + Assert::Fail(L"Failed to open gold CSV file"); + } + else { + throw std::runtime_error("Failed to open gold CSV file"); + } + return false; } std::unique_ptr pSession; if (!SetupTestEnvironment(etlFile.string(), "10000"s, pSession)) { goldCsvFile.Close(); - Assert::Fail(L"Failed to setup test environment"); - return; + if (throwOnFailure) { + Assert::Fail(L"Failed to setup test environment"); + } + else { + throw std::runtime_error("Failed to setup test environment"); + } + return false; } // Track if we encountered an assertion failure @@ -290,50 +377,177 @@ namespace EtlTests if (testCase.isExpectedFailure) { if (testPassed) { // Test passed but was expected to fail - this is noteworthy! - Logger::WriteMessage(std::format( - "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" - " Expected failure reason: {}\n" - " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", - testCase.testName, testCase.failureReason).c_str()); - // Still pass the test since it's good news + if (throwOnFailure) { + Logger::WriteMessage(std::format( + "[PASS] UNEXPECTED PASS: Test '{}' passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update GOLD_TEST_CASES to set isExpectedFailure = false\n", + testCase.testName, testCase.failureReason).c_str()); + } + // Return true because test technically passed (even though unexpected) + return true; } else { // Test failed as expected - Logger::WriteMessage(std::format( - "[FAIL] Expected failure: {}\n", - testCase.failureReason).c_str()); - // Re-throw to fail the test, but it's documented as expected - Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", - pmon::util::str::ToWide(testCase.failureReason)).c_str()); + if (throwOnFailure) { + // For individual test methods, log and assert + Logger::WriteMessage(std::format( + "[FAIL] Expected failure: {}\n", + testCase.failureReason).c_str()); + Assert::Fail(std::format(L"[EXPECTED FAILURE] {}", + pmon::util::str::ToWide(testCase.failureReason)).c_str()); + } + // Return false to indicate test failed (even though expected) + return false; } } else if (!testPassed) { - // Unexpected failure - fail the test with detailed message - Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + // Unexpected failure + if (throwOnFailure) { + Assert::Fail(pmon::util::str::ToWide(exceptionMessage).c_str()); + } + else { + // For CSV runner, throw std::runtime_error so it can be caught and counted + throw std::runtime_error(exceptionMessage); + } + return false; } - // else: test passed and wasn't expected to fail - all good! + // Test passed and wasn't expected to fail - all good! + return true; } - void RunTestFromCase(const std::string& caseName, bool useDefault = true, bool createDebugCsv = false) + // Run all test cases from a CSV file + void RunTestsFromCsv(const std::string& csvFilePath) { - auto testCase = FindTestCaseByName(caseName); - if (!testCase) { - Assert::Fail(L"Test case not found"); + // Load test cases from CSV + std::vector testCases; + try { + testCases = LoadTestCasesFromCsv(csvFilePath); + Logger::WriteMessage(std::format("Loaded {} test cases from {}\n", + testCases.size(), csvFilePath).c_str()); + } + catch (const std::exception& e) { + Assert::Fail(pmon::util::str::ToWide( + std::format("Failed to load test cases CSV: {}", e.what())).c_str()); return; } - fs::path path = useDefault ? fs::path("..") / ".." / "tests" / "gold" - : fs::path(GetAdditionalTestLocation().value_or("")); + // Statistics + int totalTests = 0; + int passedTests = 0; + int failedTests = 0; + int expectedFailures = 0; + int skippedTests = 0; + std::vector failureDetails; + + // Run each test case + for (const auto& testCase : testCases) { + // Skip if RunTest is false + if (!testCase.runTest) { + skippedTests++; + Logger::WriteMessage(std::format("[SKIP] {} (RunTest=false)\n", + testCase.testName).c_str()); + continue; + } + + totalTests++; + Logger::WriteMessage(std::format("\n=== Running Test {}/{}: {} ===\n", + totalTests, testCases.size() - skippedTests, testCase.testName).c_str()); + + // Determine test location + fs::path testPath = testCase.useAdditionalTestLocation + ? fs::path(GetAdditionalTestLocation().value_or("")) + : fs::path("..") / ".." / "tests" / "gold"; + + // Prepare debug CSV if requested + std::optional debugCsv; + if (testCase.produceDebugCsv) { + auto outputDir = testPath.string(); + auto debugCsvName = testCase.testName + "-debug"; + debugCsv = CreateCsvFile(outputDir, debugCsvName); + if (debugCsv.has_value()) { + Logger::WriteMessage(std::format(" Producing debug CSV: {}-debug.csv\n", + testCase.testName).c_str()); + } + } + + // Run the test + bool testPassed = false; + std::string errorMessage; + + try { + testPassed = RunGoldCsvTest(testCase, testPath.string(), debugCsv, false); // Returns true/false instead of throwing + } + catch (const std::exception& e) { + // Only unexpected failures throw exceptions + testPassed = false; + errorMessage = e.what(); + } + + // Handle the result + if (testPassed) { + if (testCase.isExpectedFailure) { + // Test passed but was expected to fail - this is noteworthy! + Logger::WriteMessage(std::format( + "[UNEXPECTED PASS] Test passed but was marked as expected failure!\n" + " Expected failure reason: {}\n" + " ACTION: Update CSV to set IsExpectedFailure = false\n", + testCase.failureReason).c_str()); + } + else { + Logger::WriteMessage(std::format("[PASS] {}\n", testCase.testName).c_str()); + } + passedTests++; + } + else { + // Test failed + if (testCase.isExpectedFailure) { + // Test failed as expected + expectedFailures++; + Logger::WriteMessage(std::format( + "[EXPECTED FAIL] {}\n Reason: {}\n", + testCase.testName, testCase.failureReason).c_str()); + } + else { + // Unexpected failure + failedTests++; + std::string detail = std::format("[FAIL] {}: {}", + testCase.testName, errorMessage.empty() ? "Test failed" : errorMessage); + failureDetails.push_back(detail); + Logger::WriteMessage(std::format("{}\n", detail).c_str()); + } + } - std::optional debugCsv; - if (createDebugCsv) { - auto outputDir = path.string(); - auto debugCsvName = testCase->processName + "-debug.csv"; - debugCsv = CreateCsvFile(outputDir, debugCsvName); + // Close debug CSV if it was created + if (debugCsv.has_value()) { + debugCsv->close(); + } } - RunGoldCsvTest(*testCase, path.string(), debugCsv); - if (debugCsv.has_value()) { - debugCsv->close(); + + // Print summary + Logger::WriteMessage(std::format( + "\n========================================\n" + "Test Summary\n" + "========================================\n" + "Total Test Cases in CSV: {}\n" + "Skipped (RunTest=false): {}\n" + "Tests Run: {}\n" + " Passed: {}\n" + " Failed (Unexpected): {}\n" + " Failed (Expected): {}\n" + "========================================\n", + testCases.size(), skippedTests, totalTests, + passedTests, failedTests, expectedFailures).c_str()); + + // Fail the overall test if there were unexpected failures + if (failedTests > 0) { + std::string summary = std::format( + "\n{} of {} tests failed unexpectedly:\n\n", + failedTests, totalTests); + for (const auto& detail : failureDetails) { + summary += detail + "\n"; + } + Assert::Fail(pmon::util::str::ToWide(summary).c_str()); } } public: @@ -356,6 +570,16 @@ namespace EtlTests goldCsvFile.Close(); } + // Example: Run all tests from CSV file + // This single test will run all test cases defined in the CSV + // Use the RunTest column in CSV to selectively enable/disable tests + TEST_METHOD(RunAllTestsFromCsv) + { + // CSV file is in the PresentMonAPI2Tests source directory + // Working dir is build/Debug, so go up to source tree + RunTestsFromCsv("..\\..\\IntelPresentMon\\PresentMonAPI2Tests\\test_cases.csv"); + } + TEST_METHOD(OpenServiceTest) { using namespace std::string_literals; @@ -383,174 +607,11 @@ namespace EtlTests } TEST_METHOD(OpenMockSessionTest) { - auto testCase = FindTestCaseByName("test_case_0_10792"); - auto etlFile = "..\\..\\tests\\gold\\" + testCase->etlFile; + // Simple test to verify we can create a session with an ETL file + const auto etlFile = "..\\..\\tests\\gold\\test_case_0.etl"; std::unique_ptr pSession; auto result = SetupTestEnvironment(etlFile, "10000", pSession); - Assert::IsTrue(result, L"SetupTestEnvironment failed"); - } - TEST_METHOD(Tc000v2Presenter10792) - { - RunTestFromCase("test_case_0_10792"); - } - TEST_METHOD(Tc000v2DWM1268) - { - RunTestFromCase("test_case_0_1268"); - } - - TEST_METHOD(Tc000v2Presenter8320) - { - RunTestFromCase("test_case_0_8320"); - } - TEST_METHOD(Tc000v2Presenter11648) - { - RunTestFromCase("test_case_0_11648"); - } - TEST_METHOD(Tc000v2Presenter3976) - { - RunTestFromCase("test_case_0_3976"); - } - TEST_METHOD(Tc000v2Presenter11112) - { - RunTestFromCase("test_case_0_11112"); - } - TEST_METHOD(Tc000v2Presenter2032) - { - RunTestFromCase("test_case_0_2032"); - } - TEST_METHOD(Tc000v2Presenter5988) - { - RunTestFromCase("test_case_0_5988"); - } - TEST_METHOD(Tc000v2Presenter12268) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_0_12268"); - } - TEST_METHOD(Tc000v2Presenter11100) - { - RunTestFromCase("test_case_0_11100"); - } - - TEST_METHOD(Tc001v2Dwm1564) - { - RunTestFromCase("test_case_1_1564", true, true); - } - TEST_METHOD(Tc001v2Presenter24560) - { - RunTestFromCase("test_case_1_24560"); - } - TEST_METHOD(Tc001v2devenv24944) - { - RunTestFromCase("test_case_1_24944"); - } - TEST_METHOD(Tc002v2Dwm1300) - { - RunTestFromCase("test_case_2_1300"); - } - TEST_METHOD(Tc002v2Presenter10016) - { - RunTestFromCase("test_case_2_10016"); - } - TEST_METHOD(Tc002v2Presenter5348) - { - RunTestFromCase("test_case_2_5348"); - } - TEST_METHOD(Tc002v2Presenter5220) - { - RunTestFromCase("test_case_2_5220"); - } - TEST_METHOD(Tc003v2Dwm1252) - { - RunTestFromCase("test_case_3_1252"); - } - TEST_METHOD(Tc003v2Presenter5892) - { - RunTestFromCase("test_case_3_5892"); - } - TEST_METHOD(Tc003v2Presenter10112) - { - RunTestFromCase("test_case_3_10112"); - } - TEST_METHOD(Tc003v2Presenter12980) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_3_12980"); - } - TEST_METHOD(Tc004v2Presenter5192) - { - RunTestFromCase("test_case_4_5192"); - } - TEST_METHOD(Tc004v2Presenter5236) - { - RunTestFromCase("test_case_4_5236"); - } - TEST_METHOD(Tc004v2Presenter8536) - { - RunTestFromCase("test_case_4_8536"); - } - TEST_METHOD(Tc004v2Presenter9620) - { - // This test is a sporadic failure due to timing of when the ETL session is - // finishedby the the mock presentmon session. If the ETL session finishes - // and sets the process id to not active from the mock presentmon session - // when the middleware is starting to process the NSM it will determine - // the process is not active and exit. Need to add some type of synchronization - // in mock presentmon session to not shutdown the session until notified - // by close session call. - RunTestFromCase("test_case_4_9620"); - } - TEST_METHOD(Tc004v2Dwm10376) - { - RunTestFromCase("test_case_4_10376"); - } - TEST_METHOD(Tc005v2PresentBench24892) - { - RunTestFromCase("test_case_5_24892"); - } - TEST_METHOD(Tc006CP2077) - { - RunTestFromCase("test_case_6_10796", false); - } - TEST_METHOD(Tc007CP2077) - { - RunTestFromCase("test_case_7_11320", false); - } - TEST_METHOD(Tc008ACShadows) - { - RunTestFromCase("test_case_8_6920", false); - } - TEST_METHOD(Tc009F124) - { - RunTestFromCase("test_case_9_10340", false); - } - TEST_METHOD(Tc010NarakaBladepoint) - { - RunTestFromCase("test_case_10_9888", false); - } - TEST_METHOD(Tc011NarakaBladepoint) - { - RunTestFromCase("test_case_11_1524", false); - } - TEST_METHOD(Tc012F124) - { - RunTestFromCase("test_case_12_10168", false); - } - TEST_METHOD(Tc013Dingo) - { - RunTestFromCase("test_case_13_11780", false); + Assert::IsTrue(result, L"SetupTestEnvironment failed"); } }; } \ No newline at end of file From f735da1d91c325438872b7f9b61cebf11c6fca1c Mon Sep 17 00:00:00 2001 From: markgalvan-intel Date: Mon, 10 Nov 2025 09:54:32 -0800 Subject: [PATCH 16/19] Adding in test cases csv --- .../PresentMonAPI2Tests/test_cases.csv | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 IntelPresentMon/PresentMonAPI2Tests/test_cases.csv diff --git a/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv new file mode 100644 index 00000000..7a74e9f3 --- /dev/null +++ b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv @@ -0,0 +1,36 @@ +TestName,ProcessID,ProcessName,EtlFile,GoldCsvFile,PollCount,WaitTimeSecs,IsExpectedFailure,FailureReason,UseAdditionalTestLocation,ProduceDebugCsv,RunTest +test_case_0_10792,10792,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_1268,1268,dwm.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_8320,8320,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11648,11648,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_3976,3976,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11112,11112,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_2032,2032,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_5988,5988,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_12268,12268,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_0_11100,11100,Presenter.exe,test_case_0.etl,test_case_0.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_1_1564,1564,dwm.exe,test_case_1.etl,test_case_1.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_1_24560,24560,Presenter.exe,test_case_1.etl,test_case_1.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_1_24944,24944,devenv.exe,test_case_1.etl,test_case_1.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_1300,1300,dwm.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_10016,10016,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_5348,5348,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_2_5220,5220,Presenter.exe,test_case_2.etl,test_case_2.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_1252,1252,dwm.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_5892,5892,dwm.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_10112,10112,Presenter.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_3_12980,12980,Presenter.exe,test_case_3.etl,test_case_3.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_5192,5192,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_5236,5236,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_4_8536,8536,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_9620,9620,Presenter.exe,test_case_4.etl,test_case_4.csv,10,1,FALSE,,FALSE,FALSE,TRUE +test_case_4_10376,10376,dwm.exe,test_case_4.etl,test_case_4.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_5_24892,24892,PresentBench.exe,test_case_5.etl,test_case_5.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,FALSE,FALSE,TRUE +test_case_6_10796,10796,cpLauncher.exe,test_case_6.etl,test_case_6.csv,10,1,FALSE,,TRUE,FALSE,TRUE +test_case_7_11320,11320,cpLauncher.exe,test_case_7.etl,test_case_7.csv,20,5,FALSE,,TRUE,FALSE,TRUE +test_case_8_6920,6920,scimitar_engine_win64_vs2022_llvm_fusion_dx12_px.exe,test_case_8.etl,test_case_8.csv,20,5,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_9_10340,10340,F1_24.exe,test_case_9.etl,test_case_9.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_10_9888,9888,NarakaBladepoint.exe,test_case_10.etl,test_case_10.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_11_1524,1524,NarakaBladepoint.exe,test_case_11.etl,test_case_11.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE +test_case_12_10168,10168,F1_24.exe,test_case_12.etl,test_case_12.csv,20,5,FALSE,,TRUE,FALSE,TRUE +test_case_13_11780,11780,Dingo.Main_Win64_retail.exe,test_case_13.etl,test_case_13.csv,10,1,FALSE,,TRUE,FALSE,TRUE From 98e11c0d7d232ea9a639e1f5d3760309c95e2ade Mon Sep 17 00:00:00 2001 From: "Galvan, Mark" Date: Wed, 3 Dec 2025 12:56:24 -0800 Subject: [PATCH 17/19] Removed "-in" suffix when waiting on pipe availability --- IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp index db8f0fe0..a8675611 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp +++ b/IntelPresentMon/PresentMonAPI2Tests/EtlTests.cpp @@ -266,7 +266,7 @@ namespace EtlTests "--etl-test-file"s, etlFile, bp::std_out > out, bp::std_in < in); - if (!pmon::util::pipe::DuplexPipe::WaitForAvailability(std::string(controlPipe_) + "-in", 500)) { + if (!pmon::util::pipe::DuplexPipe::WaitForAvailability(std::string(controlPipe_), 500)) { Assert::Fail(L"Timeout waiting for service control pipe"); return false; } From b5106c87049bf4122ea3c311fdc791b55dbab16d Mon Sep 17 00:00:00 2001 From: "Galvan, Mark" Date: Wed, 3 Dec 2025 12:56:55 -0800 Subject: [PATCH 18/19] Increased timeout for large test run --- IntelPresentMon/PresentMonAPI2Tests/test_cases.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv index 7a74e9f3..608967fe 100644 --- a/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv +++ b/IntelPresentMon/PresentMonAPI2Tests/test_cases.csv @@ -33,4 +33,4 @@ test_case_9_10340,10340,F1_24.exe,test_case_9.etl,test_case_9.csv,10,1,TRUE,Expe test_case_10_9888,9888,NarakaBladepoint.exe,test_case_10.etl,test_case_10.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE test_case_11_1524,1524,NarakaBladepoint.exe,test_case_11.etl,test_case_11.csv,10,1,TRUE,Expected failure - Multiple SwapChain support needed,TRUE,FALSE,TRUE test_case_12_10168,10168,F1_24.exe,test_case_12.etl,test_case_12.csv,20,5,FALSE,,TRUE,FALSE,TRUE -test_case_13_11780,11780,Dingo.Main_Win64_retail.exe,test_case_13.etl,test_case_13.csv,10,1,FALSE,,TRUE,FALSE,TRUE +test_case_13_11780,11780,Dingo.Main_Win64_retail.exe,test_case_13.etl,test_case_13.csv,10,5,FALSE,,TRUE,FALSE,TRUE From c55bbe0c8de84308f7d4efeafca1fb5e23da296a Mon Sep 17 00:00:00 2001 From: "Galvan, Mark" Date: Wed, 3 Dec 2025 13:27:49 -0800 Subject: [PATCH 19/19] Added generic template definition. --- PresentMon/CsvOutput.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PresentMon/CsvOutput.cpp b/PresentMon/CsvOutput.cpp index 1f7efe70..09593c07 100644 --- a/PresentMon/CsvOutput.cpp +++ b/PresentMon/CsvOutput.cpp @@ -186,6 +186,15 @@ void WriteCsvHeader(FILE* fp) } } +// This is a generic template that will be specialized for FrameMetrics1 and FrameMetrics. +template +void WriteCsvRow(FILE* fp, PMTraceSession const& pmSession, ProcessInfo const& processInfo, PresentEvent const& p, FrameMetricsT const& metrics) +{ + // This template should not be called directly. + // Specializations for FrameMetrics1 and FrameMetrics are provided below. + static_assert(sizeof(FrameMetricsT) == 0, "WriteCsvRow must be specialized for the given FrameMetricsT type."); +} + template<> void WriteCsvRow( FILE* fp,