From d1cd01138ee0a72d75cb1b70e8d2a5aec9944a5d Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Thu, 26 Feb 2026 19:14:44 -0600 Subject: [PATCH 01/17] Start of camera conformance tests (Assisted by Claude Code; any errors are mine.) --- MMCore/CoreCallback.cpp | 24 +- .../DeviceConformance/CameraConformance.cpp | 244 ++++++++++++++++++ MMCore/DeviceConformance/CameraConformance.h | 33 +++ .../DeviceConformance/SeqAcqTestMonitor.cpp | 98 +++++++ MMCore/DeviceConformance/SeqAcqTestMonitor.h | 71 +++++ MMCore/MMCore.cpp | 28 ++ MMCore/MMCore.h | 10 + MMCore/meson.build | 2 + 8 files changed, 509 insertions(+), 1 deletion(-) create mode 100644 MMCore/DeviceConformance/CameraConformance.cpp create mode 100644 MMCore/DeviceConformance/CameraConformance.h create mode 100644 MMCore/DeviceConformance/SeqAcqTestMonitor.cpp create mode 100644 MMCore/DeviceConformance/SeqAcqTestMonitor.h diff --git a/MMCore/CoreCallback.cpp b/MMCore/CoreCallback.cpp index 3b893853b..31ba2ba1a 100644 --- a/MMCore/CoreCallback.cpp +++ b/MMCore/CoreCallback.cpp @@ -28,6 +28,7 @@ #include "CircularBuffer.h" #include "CoreCallback.h" #include "DeviceManager.h" +#include "DeviceConformance/SeqAcqTestMonitor.h" #include "Notification.h" #include "SynchronizedConfiguration.h" @@ -227,6 +228,11 @@ int CoreCallback::InsertImage(const MM::Device* caller, const unsigned char* buf unsigned width, unsigned height, unsigned bytesPerPixel, unsigned nComponents, const char* serializedMetadata) { + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) + return monitor->OnInsertImage(); + } + Metadata origMd; if (serializedMetadata) { @@ -267,8 +273,10 @@ bool CoreCallback::InitializeImageBuffer(unsigned channels, unsigned slices, return core_->cbuf_->Initialize(w, h, pixDepth); } -int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) +int CoreCallback::AcqFinished(const MM::Device* caller, int statusCode) { + (void)statusCode; + std::shared_ptr camera; try { @@ -281,6 +289,13 @@ int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) return DEVICE_ERR; } + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) { + monitor->OnAcqFinished(); + return DEVICE_OK; + } + } + std::shared_ptr currentCamera = core_->currentCameraDevice_.lock(); @@ -340,6 +355,13 @@ int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) int CoreCallback::PrepareForAcq(const MM::Device* caller) { + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) { + monitor->OnPrepareForAcq(); + return DEVICE_OK; + } + } + if (core_->autoShutter_) { std::shared_ptr shutter = diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp new file mode 100644 index 000000000..27414e4be --- /dev/null +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -0,0 +1,244 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "CameraConformance.h" + +#include "DeviceManager.h" +#include "Devices/CameraInstance.h" +#include "Error.h" + +#include "MMDeviceConstants.h" + +#include +#include +#include +#include + +namespace mmcore { +namespace internal { + +std::string RunCameraConformanceTests( + std::shared_ptr pCam, + std::atomic& seqAcqTestMonitor, + const char* testName) { + using namespace std::chrono; + + const MM::Device* rawCam = pCam->GetRawPtr(); + const auto testTimeout = seconds(10); + const auto postErrorDelay = seconds(2); + + // RAII guard: stop camera first (joining its thread), then clear atomic. + struct MonitorGuard { + std::atomic& atom; + std::shared_ptr& cam; + ~MonitorGuard() { + try { + DeviceModuleLockGuard g(cam); + if (cam->IsCapturing()) + cam->StopSequenceAcquisition(); + } catch (...) {} + atom.store(nullptr, std::memory_order_release); + } + }; + + auto startFinite = [&](long numImages, double intervalMs) { + DeviceModuleLockGuard guard(pCam); + int nRet = pCam->StartSequenceAcquisition(numImages, intervalMs, false); + if (nRet != DEVICE_OK) + throw CMMError("Camera failed to start finite sequence acquisition"); + }; + + auto startContinuous = [&](double intervalMs) { + DeviceModuleLockGuard guard(pCam); + int nRet = pCam->StartSequenceAcquisition(intervalMs); + if (nRet != DEVICE_OK) + throw CMMError("Camera failed to start continuous sequence acquisition"); + }; + + auto stopCamera = [&]() { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) + pCam->StopSequenceAcquisition(); + }; + + std::ostringstream report; + int totalTests = 0; + int passedTests = 0; + + // --- Test 1: seq-prepare-before-insert --- + auto testPrepareBeforeInsert = [&]() { + report << "--- seq-prepare-before-insert ---\n"; + ++totalTests; + + SeqAcqTestMonitor monitor(rawCam); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + startFinite(5, 0.0); + monitor.WaitForInsertImageCount(5, testTimeout); + + bool pass = true; + if (!monitor.PrepareForAcqCalled()) { + report << "FAIL: PrepareForAcq was not called\n"; + pass = false; + } else if (!monitor.PrepareBeforeFirstInsert()) { + report << "FAIL: PrepareForAcq was called after InsertImage\n"; + pass = false; + } else { + report << "PASS: PrepareForAcq called before first InsertImage\n"; + } + if (pass) ++passedTests; + }; + + // --- Test 2: seq-finished-after-count --- + auto testFinishedAfterCount = [&]() { + report << "--- seq-finished-after-count ---\n"; + ++totalTests; + + SeqAcqTestMonitor monitor(rawCam); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + startFinite(5, 0.0); + monitor.WaitForInsertImageCount(5, testTimeout); + + bool pass = false; + if (monitor.WaitForAcqFinished(testTimeout)) { + report << "PASS: AcqFinished called after finite sequence completed\n"; + pass = true; + } else { + report << "FAIL: AcqFinished not called after finite sequence (5 frames)\n"; + stopCamera(); + if (monitor.WaitForAcqFinished(testTimeout)) { + report << " (AcqFinished was called after stopSequenceAcquisition)\n"; + } else { + report << " (AcqFinished was not called even after stopSequenceAcquisition)\n"; + } + } + if (pass) ++passedTests; + }; + + // --- Tests 3-6 share a pattern: error/overflow injection --- + auto testFinishedOnError = [&](const char* slug, int errorCode, + const char* errorName, bool continuous) { + report << "--- " << slug << " ---\n"; + ++totalTests; + + SeqAcqTestMonitor monitor(rawCam); + monitor.SetErrorInjection(errorCode, 3); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + if (continuous) + startContinuous(0.0); + else + startFinite(1000000, 0.0); + + monitor.WaitForInsertImageCount(3, testTimeout); + + bool pass = true; + + if (monitor.WaitForAcqFinished(testTimeout)) { + report << "PASS: AcqFinished called after " << errorName << "\n"; + } else { + report << "FAIL: AcqFinished not called after " << errorName << "\n"; + pass = false; + stopCamera(); + if (monitor.WaitForAcqFinished(testTimeout)) { + report << " (AcqFinished was called after stopSequenceAcquisition)\n"; + } else { + report << " (AcqFinished was not called even after stopSequenceAcquisition)\n"; + } + } + + std::this_thread::sleep_for(postErrorDelay); + int afterError = monitor.InsertImageCountAfterError(); + if (afterError > 1) { + report << "FAIL: " << (afterError - 1) + << " InsertImage call(s) after error return\n"; + pass = false; + } else { + report << "PASS: No further InsertImage calls after error\n"; + } + + if (pass) ++passedTests; + }; + + // --- Test 3: seq-finished-on-error-finite --- + auto testFinishedOnErrorFinite = [&]() { + testFinishedOnError("seq-finished-on-error-finite", + DEVICE_ERR, "DEVICE_ERR", false); + }; + + // --- Test 4: seq-finished-on-error-continuous --- + auto testFinishedOnErrorContinuous = [&]() { + testFinishedOnError("seq-finished-on-error-continuous", + DEVICE_ERR, "DEVICE_ERR", true); + }; + + // --- Test 5: seq-finished-on-overflow-finite --- + auto testFinishedOnOverflowFinite = [&]() { + testFinishedOnError("seq-finished-on-overflow-finite", + DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); + }; + + // --- Test 6: seq-finished-on-overflow-continuous --- + auto testFinishedOnOverflowContinuous = [&]() { + testFinishedOnError("seq-finished-on-overflow-continuous", + DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); + }; + + struct TestEntry { + const char* slug; + std::function func; + }; + TestEntry tests[] = { + {"seq-prepare-before-insert", testPrepareBeforeInsert}, + {"seq-finished-after-count", testFinishedAfterCount}, + {"seq-finished-on-error-finite", testFinishedOnErrorFinite}, + {"seq-finished-on-error-continuous", testFinishedOnErrorContinuous}, + {"seq-finished-on-overflow-finite", testFinishedOnOverflowFinite}, + {"seq-finished-on-overflow-continuous", testFinishedOnOverflowContinuous}, + }; + + std::string selectedTest; + if (testName && testName[0] != '\0') + selectedTest = testName; + + if (!selectedTest.empty()) { + bool found = false; + for (const auto& t : tests) { + if (selectedTest == t.slug) { + found = true; + break; + } + } + if (!found) { + throw CMMError("Unknown camera test: " + selectedTest); + } + } + + for (const auto& t : tests) { + if (!selectedTest.empty() && selectedTest != t.slug) + continue; + t.func(); + report << "\n"; + } + + report << "=== Summary: " << passedTests << " / " << totalTests + << " tests passed ===\n"; + return report.str(); +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h new file mode 100644 index 000000000..ed022200a --- /dev/null +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -0,0 +1,33 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "SeqAcqTestMonitor.h" + +#include +#include +#include + +namespace mmcore { +namespace internal { + +class CameraInstance; + +std::string RunCameraConformanceTests( + std::shared_ptr camera, + std::atomic& testMonitor, + const char* testName); + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp new file mode 100644 index 000000000..42f3587b7 --- /dev/null +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp @@ -0,0 +1,98 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "SeqAcqTestMonitor.h" + +namespace mmcore { +namespace internal { + +void SeqAcqTestMonitor::SetErrorInjection(int errorCode, + int afterSuccessfulCount) { + std::lock_guard lock(mutex_); + injectErrorCode_ = errorCode; + injectAfterCount_ = afterSuccessfulCount; +} + +void SeqAcqTestMonitor::OnPrepareForAcq() { + std::lock_guard lock(mutex_); + prepareForAcqCalled_ = true; + if (insertImageCount_ == 0) + prepareBeforeFirstInsert_ = true; +} + +int SeqAcqTestMonitor::OnInsertImage() { + std::lock_guard lock(mutex_); + if (errorInjected_) { + ++insertImageCountAfterError_; + cv_.notify_all(); + return injectErrorCode_; + } + if (injectErrorCode_ != DEVICE_OK && + insertImageCount_ >= injectAfterCount_) { + errorInjected_ = true; + ++insertImageCountAfterError_; + cv_.notify_all(); + return injectErrorCode_; + } + ++insertImageCount_; + cv_.notify_all(); + return DEVICE_OK; +} + +void SeqAcqTestMonitor::OnAcqFinished() { + std::lock_guard lock(mutex_); + acqFinishedCalled_ = true; + cv_.notify_all(); +} + +bool SeqAcqTestMonitor::WaitForInsertImageCount(int n, + std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, + [&] { return insertImageCount_ >= n || errorInjected_; }); +} + +bool SeqAcqTestMonitor::WaitForAcqFinished( + std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, + [&] { return acqFinishedCalled_; }); +} + +bool SeqAcqTestMonitor::PrepareForAcqCalled() const { + std::lock_guard lock(mutex_); + return prepareForAcqCalled_; +} + +int SeqAcqTestMonitor::InsertImageCount() const { + std::lock_guard lock(mutex_); + return insertImageCount_; +} + +bool SeqAcqTestMonitor::AcqFinishedCalled() const { + std::lock_guard lock(mutex_); + return acqFinishedCalled_; +} + +bool SeqAcqTestMonitor::PrepareBeforeFirstInsert() const { + std::lock_guard lock(mutex_); + return prepareBeforeFirstInsert_; +} + +int SeqAcqTestMonitor::InsertImageCountAfterError() const { + std::lock_guard lock(mutex_); + return insertImageCountAfterError_; +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.h b/MMCore/DeviceConformance/SeqAcqTestMonitor.h new file mode 100644 index 000000000..135fe8ba5 --- /dev/null +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.h @@ -0,0 +1,71 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "MMDevice.h" +#include "MMDeviceConstants.h" + +#include +#include +#include + +namespace mmcore { +namespace internal { + +class SeqAcqTestMonitor { +public: + explicit SeqAcqTestMonitor(const MM::Device* target) : target_(target) {} + + SeqAcqTestMonitor(const SeqAcqTestMonitor&) = delete; + SeqAcqTestMonitor& operator=(const SeqAcqTestMonitor&) = delete; + + bool IsMonitoring(const MM::Device* caller) const { + return caller == target_; + } + + void SetErrorInjection(int errorCode, int afterSuccessfulCount); + + void OnPrepareForAcq(); + int OnInsertImage(); + void OnAcqFinished(); + + bool WaitForInsertImageCount(int n, std::chrono::milliseconds timeout); + bool WaitForAcqFinished(std::chrono::milliseconds timeout); + + bool PrepareForAcqCalled() const; + int InsertImageCount() const; + bool AcqFinishedCalled() const; + bool PrepareBeforeFirstInsert() const; + int InsertImageCountAfterError() const; + +private: + const MM::Device* const target_; + + mutable std::mutex mutex_; + std::condition_variable cv_; + + bool prepareForAcqCalled_ = false; + int insertImageCount_ = 0; + bool acqFinishedCalled_ = false; + + bool prepareBeforeFirstInsert_ = false; + + int injectErrorCode_ = DEVICE_OK; + int injectAfterCount_ = 0; + bool errorInjected_ = false; + int insertImageCountAfterError_ = 0; +}; + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index a1e33dbb1..0d2c885cf 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -51,6 +51,7 @@ #include "MMEventCallback.h" #include "NotificationQueue.h" #include "PluginManager.h" +#include "DeviceConformance/CameraConformance.h" #include "SynchronizedConfiguration.h" #include "DeviceUtils.h" @@ -3087,6 +3088,33 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) return pCam->IsCapturing(); }; +/** + * Run behavioral tests on a camera device adapter and return a text report. + * + * The tests exercise the sequence acquisition callback protocol + * (PrepareForAcq, InsertImage, AcqFinished). + * + * @param cameraLabel Label of the camera to test (must be loaded and + * initialized, and not currently acquiring). + * @param testName Slug name of a single test to run, or null/empty to run + * all tests. + * @return A human-readable text report. + */ +std::string CMMCore::runCameraConformanceTests(const char* cameraLabel, + const char* testName) MMCORE_LEGACY_THROW(CMMError) +{ + auto pCam = deviceManager_->GetDeviceOfType(cameraLabel); + { + mmi::DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) + throw CMMError( + getCoreErrorText(MMERR_NotAllowedDuringSequenceAcquisition), + MMERR_NotAllowedDuringSequenceAcquisition); + } + + return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, testName); +} + /** * Gets the last image from the circular buffer. * Returns 0 if the buffer is empty. diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index 2da918627..fddb8b156 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -56,6 +56,7 @@ #include "MMDevice.h" #include "MMDeviceConstants.h" +#include #include #include #include @@ -95,6 +96,7 @@ namespace internal { class DeviceManager; class LogManager; class NotificationQueue; + class SeqAcqTestMonitor; } // namespace internal } // namespace mmcore @@ -677,6 +679,12 @@ class CMMCore std::vector getLoadedPeripheralDevices(const char* hubLabel) MMCORE_LEGACY_THROW(CMMError); ///@} + /** \name Device conformance testing. */ + ///@{ + std::string runCameraConformanceTests(const char* cameraLabel, + const char* testName = nullptr) MMCORE_LEGACY_THROW(CMMError); + ///@} + #if !defined(SWIGJAVA) && !defined(SWIGPYTHON) /** \name Testing */ ///@{ @@ -720,6 +728,8 @@ class CMMCore std::unique_ptr cbuf_; std::unique_ptr callback_; + std::atomic seqAcqTestMonitor_{nullptr}; + std::shared_ptr pluginManager_; std::shared_ptr deviceManager_; std::map errorText_; diff --git a/MMCore/meson.build b/MMCore/meson.build index 0ccb60e1a..0b13bee67 100644 --- a/MMCore/meson.build +++ b/MMCore/meson.build @@ -62,6 +62,8 @@ mmcore_sources = files( 'LogManager.cpp', 'MMCore.cpp', 'PluginManager.cpp', + 'DeviceConformance/CameraConformance.cpp', + 'DeviceConformance/SeqAcqTestMonitor.cpp', 'Semaphore.cpp', 'Task.cpp', 'TaskSet.cpp', From c964d55d3239c5b4a649f79e93bb334db239266a Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Thu, 26 Feb 2026 20:02:26 -0600 Subject: [PATCH 02/17] Use JSON for conformance test results (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 168 +++++++++++++----- MMCore/DeviceConformance/CameraConformance.h | 18 +- MMCore/MMCore.cpp | 7 +- MMCore/meson.build | 1 + MMCore/subprojects/.gitignore | 1 + MMCore/subprojects/nlohmann_json.wrap | 11 ++ 6 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 MMCore/subprojects/nlohmann_json.wrap diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 27414e4be..91e3a5cc3 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -19,20 +19,67 @@ #include "MMDeviceConstants.h" +#include + #include +#include #include +#include #include #include namespace mmcore { namespace internal { +namespace { + +std::string FormatISO8601(std::chrono::system_clock::time_point tp) { + auto time = std::chrono::system_clock::to_time_t(tp); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &time); +#else + gmtime_r(&time, &tm); +#endif + std::ostringstream ss; + ss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); + return ss.str(); +} + +nlohmann::json AssertionToJson(const AssertionResult& a) { + nlohmann::json j; + j["passed"] = a.passed; + j["message"] = a.message; + if (!a.details.empty()) + j["details"] = a.details; + return j; +} + +nlohmann::json TestToJson(const TestResult& t) { + nlohmann::json j; + j["name"] = t.name; + j["passed"] = t.passed; + nlohmann::json assertions = nlohmann::json::array(); + for (const auto& a : t.assertions) + assertions.push_back(AssertionToJson(a)); + j["assertions"] = assertions; + return j; +} + +} // anonymous namespace + std::string RunCameraConformanceTests( std::shared_ptr pCam, std::atomic& seqAcqTestMonitor, - const char* testName) { + const char* testName, + const std::string& deviceLabel, + const std::string& deviceName, + const std::string& adapterName) { using namespace std::chrono; + const auto startTime = system_clock::now(); + const auto startSteady = steady_clock::now(); + const MM::Device* rawCam = pCam->GetRawPtr(); const auto testTimeout = seconds(10); const auto postErrorDelay = seconds(2); @@ -71,14 +118,11 @@ std::string RunCameraConformanceTests( pCam->StopSequenceAcquisition(); }; - std::ostringstream report; - int totalTests = 0; - int passedTests = 0; + std::vector results; - // --- Test 1: seq-prepare-before-insert --- auto testPrepareBeforeInsert = [&]() { - report << "--- seq-prepare-before-insert ---\n"; - ++totalTests; + TestResult result; + result.name = "seq-prepare-before-insert"; SeqAcqTestMonitor monitor(rawCam); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -87,23 +131,24 @@ std::string RunCameraConformanceTests( startFinite(5, 0.0); monitor.WaitForInsertImageCount(5, testTimeout); - bool pass = true; if (!monitor.PrepareForAcqCalled()) { - report << "FAIL: PrepareForAcq was not called\n"; - pass = false; + result.assertions.push_back( + {false, "PrepareForAcq was not called", {}}); } else if (!monitor.PrepareBeforeFirstInsert()) { - report << "FAIL: PrepareForAcq was called after InsertImage\n"; - pass = false; + result.assertions.push_back( + {false, "PrepareForAcq was called after InsertImage", {}}); } else { - report << "PASS: PrepareForAcq called before first InsertImage\n"; + result.assertions.push_back( + {true, "PrepareForAcq called before first InsertImage", {}}); } - if (pass) ++passedTests; + + result.passed = result.assertions.back().passed; + results.push_back(std::move(result)); }; - // --- Test 2: seq-finished-after-count --- auto testFinishedAfterCount = [&]() { - report << "--- seq-finished-after-count ---\n"; - ++totalTests; + TestResult result; + result.name = "seq-finished-after-count"; SeqAcqTestMonitor monitor(rawCam); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -112,27 +157,32 @@ std::string RunCameraConformanceTests( startFinite(5, 0.0); monitor.WaitForInsertImageCount(5, testTimeout); - bool pass = false; if (monitor.WaitForAcqFinished(testTimeout)) { - report << "PASS: AcqFinished called after finite sequence completed\n"; - pass = true; + result.assertions.push_back( + {true, "AcqFinished called after finite sequence completed", {}}); } else { - report << "FAIL: AcqFinished not called after finite sequence (5 frames)\n"; + AssertionResult a; + a.passed = false; + a.message = "AcqFinished not called after finite sequence (5 frames)"; stopCamera(); if (monitor.WaitForAcqFinished(testTimeout)) { - report << " (AcqFinished was called after stopSequenceAcquisition)\n"; + a.details.push_back( + "AcqFinished was called after stopSequenceAcquisition"); } else { - report << " (AcqFinished was not called even after stopSequenceAcquisition)\n"; + a.details.push_back( + "AcqFinished was not called even after stopSequenceAcquisition"); } + result.assertions.push_back(std::move(a)); } - if (pass) ++passedTests; + + result.passed = result.assertions.back().passed; + results.push_back(std::move(result)); }; - // --- Tests 3-6 share a pattern: error/overflow injection --- auto testFinishedOnError = [&](const char* slug, int errorCode, const char* errorName, bool continuous) { - report << "--- " << slug << " ---\n"; - ++totalTests; + TestResult result; + result.name = slug; SeqAcqTestMonitor monitor(rawCam); monitor.SetErrorInjection(errorCode, 3); @@ -149,50 +199,55 @@ std::string RunCameraConformanceTests( bool pass = true; if (monitor.WaitForAcqFinished(testTimeout)) { - report << "PASS: AcqFinished called after " << errorName << "\n"; + result.assertions.push_back( + {true, std::string("AcqFinished called after ") + errorName, {}}); } else { - report << "FAIL: AcqFinished not called after " << errorName << "\n"; + AssertionResult a; + a.passed = false; + a.message = std::string("AcqFinished not called after ") + errorName; pass = false; stopCamera(); if (monitor.WaitForAcqFinished(testTimeout)) { - report << " (AcqFinished was called after stopSequenceAcquisition)\n"; + a.details.push_back( + "AcqFinished was called after stopSequenceAcquisition"); } else { - report << " (AcqFinished was not called even after stopSequenceAcquisition)\n"; + a.details.push_back( + "AcqFinished was not called even after stopSequenceAcquisition"); } + result.assertions.push_back(std::move(a)); } std::this_thread::sleep_for(postErrorDelay); int afterError = monitor.InsertImageCountAfterError(); if (afterError > 1) { - report << "FAIL: " << (afterError - 1) - << " InsertImage call(s) after error return\n"; + result.assertions.push_back( + {false, std::to_string(afterError - 1) + + " InsertImage call(s) after error return", {}}); pass = false; } else { - report << "PASS: No further InsertImage calls after error\n"; + result.assertions.push_back( + {true, "No further InsertImage calls after error", {}}); } - if (pass) ++passedTests; + result.passed = pass; + results.push_back(std::move(result)); }; - // --- Test 3: seq-finished-on-error-finite --- auto testFinishedOnErrorFinite = [&]() { testFinishedOnError("seq-finished-on-error-finite", DEVICE_ERR, "DEVICE_ERR", false); }; - // --- Test 4: seq-finished-on-error-continuous --- auto testFinishedOnErrorContinuous = [&]() { testFinishedOnError("seq-finished-on-error-continuous", DEVICE_ERR, "DEVICE_ERR", true); }; - // --- Test 5: seq-finished-on-overflow-finite --- auto testFinishedOnOverflowFinite = [&]() { testFinishedOnError("seq-finished-on-overflow-finite", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); }; - // --- Test 6: seq-finished-on-overflow-continuous --- auto testFinishedOnOverflowContinuous = [&]() { testFinishedOnError("seq-finished-on-overflow-continuous", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); @@ -232,12 +287,39 @@ std::string RunCameraConformanceTests( if (!selectedTest.empty() && selectedTest != t.slug) continue; t.func(); - report << "\n"; } - report << "=== Summary: " << passedTests << " / " << totalTests - << " tests passed ===\n"; - return report.str(); + const auto endSteady = steady_clock::now(); + double durationMs = + duration_cast>(endSteady - startSteady) + .count(); + + int passedCount = 0; + for (const auto& r : results) + if (r.passed) + ++passedCount; + + nlohmann::json testsJson = nlohmann::json::array(); + for (const auto& r : results) + testsJson.push_back(TestToJson(r)); + + nlohmann::json j; + j["version"] = 1; + j["timestamp"] = FormatISO8601(startTime); + j["device"] = { + {"label", deviceLabel}, + {"name", deviceName}, + {"library", adapterName}, + }; + j["deviceType"] = "Camera"; + j["tests"] = testsJson; + j["summary"] = { + {"total", static_cast(results.size())}, + {"passed", passedCount}, + {"durationMs", durationMs}, + }; + + return j.dump(2); } } // namespace internal diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h index ed022200a..6edbf4fd8 100644 --- a/MMCore/DeviceConformance/CameraConformance.h +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -18,16 +18,32 @@ #include #include #include +#include namespace mmcore { namespace internal { class CameraInstance; +struct AssertionResult { + bool passed; + std::string message; + std::vector details; +}; + +struct TestResult { + std::string name; + bool passed; + std::vector assertions; +}; + std::string RunCameraConformanceTests( std::shared_ptr camera, std::atomic& testMonitor, - const char* testName); + const char* testName, + const std::string& deviceLabel, + const std::string& deviceName, + const std::string& adapterName); } // namespace internal } // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index 0d2c885cf..df21b0b38 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -3089,7 +3089,7 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) }; /** - * Run behavioral tests on a camera device adapter and return a text report. + * Run behavioral tests on a camera device adapter and return a JSON report. * * The tests exercise the sequence acquisition callback protocol * (PrepareForAcq, InsertImage, AcqFinished). @@ -3098,7 +3098,7 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) * initialized, and not currently acquiring). * @param testName Slug name of a single test to run, or null/empty to run * all tests. - * @return A human-readable text report. + * @return A JSON string containing the test results. */ std::string CMMCore::runCameraConformanceTests(const char* cameraLabel, const char* testName) MMCORE_LEGACY_THROW(CMMError) @@ -3112,7 +3112,8 @@ std::string CMMCore::runCameraConformanceTests(const char* cameraLabel, MMERR_NotAllowedDuringSequenceAcquisition); } - return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, testName); + return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, testName, + cameraLabel, pCam->GetName(), pCam->GetAdapterModule()->GetName()); } /** diff --git a/MMCore/meson.build b/MMCore/meson.build index 0b13bee67..5c46a9888 100644 --- a/MMCore/meson.build +++ b/MMCore/meson.build @@ -98,6 +98,7 @@ mmcore_lib = static_library( include_directories: mmcore_include_dir, dependencies: [ mmdevice_dep, + dependency('nlohmann_json'), dependency('threads'), dependency('dl', required: false), ], diff --git a/MMCore/subprojects/.gitignore b/MMCore/subprojects/.gitignore index d12f32e5d..2dae20e18 100644 --- a/MMCore/subprojects/.gitignore +++ b/MMCore/subprojects/.gitignore @@ -13,3 +13,4 @@ # Do not ignore wraps we provide !/catch2.wrap !/mmdevice.wrap +!/nlohmann_json.wrap diff --git a/MMCore/subprojects/nlohmann_json.wrap b/MMCore/subprojects/nlohmann_json.wrap new file mode 100644 index 000000000..f9bb22de1 --- /dev/null +++ b/MMCore/subprojects/nlohmann_json.wrap @@ -0,0 +1,11 @@ +[wrap-file] +directory = nlohmann_json-3.12.0 +lead_directory_missing = true +source_url = https://github.com/nlohmann/json/releases/download/v3.12.0/include.zip +source_filename = nlohmann_json-3.12.0.zip +source_hash = b8cb0ef2dd7f57f18933997c9934bb1fa962594f701cd5a8d3c2c80541559372 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/nlohmann_json_3.12.0-1/nlohmann_json-3.12.0.zip +wrapdb_version = 3.12.0-1 + +[provide] +nlohmann_json = nlohmann_json_dep From 8bf1353236b5bdfd75297c7feb3c3666e9a50b60 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Thu, 26 Feb 2026 22:16:03 -0600 Subject: [PATCH 03/17] Unit tests for camera conformance tests (Assisted by Claude Code; any errors are mine.) --- MMCore/meson.build | 4 +- MMCore/unittest/CameraConformance-Tests.cpp | 227 ++++++++++++++++++++ MMCore/unittest/meson.build | 4 +- 3 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 MMCore/unittest/CameraConformance-Tests.cpp diff --git a/MMCore/meson.build b/MMCore/meson.build index 5c46a9888..efe02f6dc 100644 --- a/MMCore/meson.build +++ b/MMCore/meson.build @@ -27,6 +27,8 @@ mmdevice_proj = subproject( ) mmdevice_dep = mmdevice_proj.get_variable('mmdevice_dep') +nlohmann_json_dep = dependency('nlohmann_json') + mmcore_sources = files( 'CircularBuffer.cpp', 'Configuration.cpp', @@ -98,7 +100,7 @@ mmcore_lib = static_library( include_directories: mmcore_include_dir, dependencies: [ mmdevice_dep, - dependency('nlohmann_json'), + nlohmann_json_dep, dependency('threads'), dependency('dl', required: false), ], diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp new file mode 100644 index 000000000..6ee42f732 --- /dev/null +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -0,0 +1,227 @@ +#include + +#include "DeviceBase.h" +#include "MMCore.h" +#include "MMDeviceConstants.h" +#include "MockDeviceUtils.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +bool TestPassed(const nlohmann::json& results, const std::string& testName) { + for (const auto& t : results["tests"]) { + if (t["name"] == testName) + return t["passed"].get(); + } + throw std::runtime_error("Test not found: " + testName); +} + +struct ConfigurableAsyncCamera : CCameraBase { + std::string name = "ConfigurableAsyncCamera"; + unsigned width = 64; + unsigned height = 64; + unsigned bytesPerPixel = 1; + unsigned nComponents = 1; + unsigned bitDepth = 8; + int binning = 1; + double exposure = 10.0; + + bool callPrepareForAcq = true; + bool callAcqFinished = true; + bool checkInsertImageReturn = true; + + int Initialize() override { return DEVICE_OK; } + int Shutdown() override { return DEVICE_OK; } + bool Busy() override { return false; } + void GetName(char* buf) const override { + CDeviceUtils::CopyLimitedString(buf, name.c_str()); + } + + int SnapImage() override { + imgBuf_.assign( + static_cast(width) * height * bytesPerPixel, 0); + return DEVICE_OK; + } + const unsigned char* GetImageBuffer() override { + return imgBuf_.data(); + } + long GetImageBufferSize() const override { + return static_cast(width) * height * bytesPerPixel; + } + unsigned GetImageWidth() const override { return width; } + unsigned GetImageHeight() const override { return height; } + unsigned GetImageBytesPerPixel() const override { return bytesPerPixel; } + unsigned GetNumberOfComponents() const override { return nComponents; } + unsigned GetBitDepth() const override { return bitDepth; } + int GetBinning() const override { return binning; } + int SetBinning(int b) override { binning = b; return DEVICE_OK; } + void SetExposure(double e) override { exposure = e; } + double GetExposure() const override { return exposure; } + int SetROI(unsigned, unsigned, unsigned, unsigned) override { + return DEVICE_OK; + } + int GetROI(unsigned& x, unsigned& y, unsigned& w, unsigned& h) override { + x = 0; y = 0; w = width; h = height; + return DEVICE_OK; + } + int ClearROI() override { return DEVICE_OK; } + int IsExposureSequenceable(bool& seq) const override { + seq = false; + return DEVICE_OK; + } + + int StartSequenceAcquisition(long numImages, double, bool) override { + // Thread may be left over from previous (unstopped) run + if (thread_.joinable()) + thread_.join(); + if (callPrepareForAcq) + GetCoreCallback()->PrepareForAcq(this); + { + std::lock_guard lk(mu_); + running_ = true; + stopRequested_ = false; + } + thread_ = std::thread([this, numImages] { + AcqThread(numImages); + }); + return DEVICE_OK; + } + + int StartSequenceAcquisition(double intervalMs) override { + return StartSequenceAcquisition(-1, intervalMs, false); + } + + ~ConfigurableAsyncCamera() { + { + std::lock_guard lk(mu_); + stopRequested_ = true; + } + cv_.notify_one(); + if (thread_.joinable()) + thread_.join(); + } + + int StopSequenceAcquisition() override { + { + std::lock_guard lk(mu_); + stopRequested_ = true; + } + cv_.notify_one(); + if (thread_.joinable()) + thread_.join(); + return DEVICE_OK; + } + + bool IsCapturing() override { + std::lock_guard lk(mu_); + return running_; + } + +private: + void AcqThread(long numImages) { + std::vector buf( + static_cast(width) * height * bytesPerPixel, 0); + long count = 0; + while (numImages < 0 || count < numImages) { + { + std::lock_guard lk(mu_); + if (stopRequested_) + break; + } + int ret = GetCoreCallback()->InsertImage(this, buf.data(), + width, height, bytesPerPixel, nComponents); + if (checkInsertImageReturn && ret != DEVICE_OK) + break; + ++count; + { + std::unique_lock lk(mu_); + cv_.wait_for(lk, std::chrono::microseconds(100), + [this] { return stopRequested_; }); + if (stopRequested_) + break; + } + } + if (callAcqFinished) + GetCoreCallback()->AcqFinished(this, 0); + { + std::lock_guard lk(mu_); + running_ = false; + } + } + + bool running_ = false; + bool stopRequested_ = false; + std::mutex mu_; + std::condition_variable cv_; + std::thread thread_; + std::vector imgBuf_; +}; + +} // anonymous namespace + +TEST_CASE("Conformant camera passes all conformance tests", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runCameraConformanceTests("cam")); + for (const auto& test : results["tests"]) { + INFO("Test: " << test["name"].get()); + CHECK(test["passed"].get()); + } + CHECK(results["summary"]["passed"].get() == + results["summary"]["total"].get()); +} + +TEST_CASE("Missing PrepareForAcq is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.callPrepareForAcq = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runCameraConformanceTests("cam", "seq-prepare-before-insert")); + CHECK_FALSE(TestPassed(results, "seq-prepare-before-insert")); +} + +TEST_CASE("Missing AcqFinished is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.callAcqFinished = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runCameraConformanceTests("cam", "seq-finished-after-count")); + CHECK_FALSE(TestPassed(results, "seq-finished-after-count")); +} + +TEST_CASE("Ignoring InsertImage error return is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.checkInsertImageReturn = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runCameraConformanceTests("cam", "seq-finished-on-error-finite")); + CHECK_FALSE(TestPassed(results, "seq-finished-on-error-finite")); +} diff --git a/MMCore/unittest/meson.build b/MMCore/unittest/meson.build index 7245e693f..8849bf78c 100644 --- a/MMCore/unittest/meson.build +++ b/MMCore/unittest/meson.build @@ -11,6 +11,7 @@ catch2_with_main_dep = dependency( mmcore_test_sources = files( 'APIError-Tests.cpp', + 'CameraConformance-Tests.cpp', 'CircularBuffer-Tests.cpp', 'CoreCreateDestroy-Tests.cpp', 'CoreProperties-Tests.cpp', @@ -36,10 +37,11 @@ mmcore_test_exe = executable( dependencies: [ mmdevice_dep, catch2_with_main_dep, + nlohmann_json_dep, ], cpp_args: [ '-D_CRT_SECURE_NO_WARNINGS', # TODO Eliminate the need ], ) -test('MMCore tests', mmcore_test_exe) +test('MMCore tests', mmcore_test_exe, timeout: 120) From 08e7634ca08597e3429d0d00465a46cbb3b62330 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Wed, 11 Mar 2026 10:38:46 -0500 Subject: [PATCH 04/17] Timeout config for conformance tests We need to use short timeouts for unit tests of the conformance tests. (Before this, the unit tests took 45 s to run.) Keep out of public API for now, because we may gain more tuning knobs for conformance tests. Eventually we should probably allow configuration by application so that, e.g., unusually slow devices can be tested. (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 5 ++-- MMCore/DeviceConformance/CameraConformance.h | 2 ++ .../DeviceConformance/ConformanceTestConfig.h | 27 +++++++++++++++++++ MMCore/MMCore.cpp | 18 ++++++++++++- MMCore/MMCore.h | 4 +++ MMCore/unittest/CameraConformance-Tests.cpp | 10 +++++++ 6 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 MMCore/DeviceConformance/ConformanceTestConfig.h diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 91e3a5cc3..931563d67 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -71,6 +71,7 @@ nlohmann::json TestToJson(const TestResult& t) { std::string RunCameraConformanceTests( std::shared_ptr pCam, std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config, const char* testName, const std::string& deviceLabel, const std::string& deviceName, @@ -81,8 +82,8 @@ std::string RunCameraConformanceTests( const auto startSteady = steady_clock::now(); const MM::Device* rawCam = pCam->GetRawPtr(); - const auto testTimeout = seconds(10); - const auto postErrorDelay = seconds(2); + const auto testTimeout = config.positiveTimeout; + const auto postErrorDelay = config.negativeTimeout; // RAII guard: stop camera first (joining its thread), then clear atomic. struct MonitorGuard { diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h index 6edbf4fd8..a8f1d039b 100644 --- a/MMCore/DeviceConformance/CameraConformance.h +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -13,6 +13,7 @@ #pragma once +#include "ConformanceTestConfig.h" #include "SeqAcqTestMonitor.h" #include @@ -40,6 +41,7 @@ struct TestResult { std::string RunCameraConformanceTests( std::shared_ptr camera, std::atomic& testMonitor, + const ConformanceTestConfig& config, const char* testName, const std::string& deviceLabel, const std::string& deviceName, diff --git a/MMCore/DeviceConformance/ConformanceTestConfig.h b/MMCore/DeviceConformance/ConformanceTestConfig.h new file mode 100644 index 000000000..ca5d1696b --- /dev/null +++ b/MMCore/DeviceConformance/ConformanceTestConfig.h @@ -0,0 +1,27 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include + +namespace mmcore { +namespace internal { + +struct ConformanceTestConfig { + std::chrono::milliseconds positiveTimeout{10000}; + std::chrono::milliseconds negativeTimeout{2000}; +}; + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index df21b0b38..48ec48d6d 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -135,6 +135,7 @@ CMMCore::CMMCore() : cbuf_(std::make_unique( (sizeof(void*) > 4) ? 250u : 25u)), callback_(std::make_unique(this)), + conformanceTestConfig_(std::make_unique()), pluginManager_(std::make_shared()), deviceManager_(std::make_shared()), stateCache_(std::make_unique()) @@ -3112,10 +3113,25 @@ std::string CMMCore::runCameraConformanceTests(const char* cameraLabel, MMERR_NotAllowedDuringSequenceAcquisition); } - return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, testName, + return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, + *conformanceTestConfig_, testName, cameraLabel, pCam->GetName(), pCam->GetAdapterModule()->GetName()); } +/** + * \brief Testing only: configure conformance tests + * + * This function is designed for unit testing of MMCore itself, and its + * interface is subject to change. It is also not designed for language + * bindings (Java, Python) in mind (at least for now). + * + * Do not use this in production code, for now. + */ +void CMMCore::setConformanceTestConfig( + const mmcore::internal::ConformanceTestConfig& config) { + *conformanceTestConfig_ = config; +} + /** * Gets the last image from the circular buffer. * Returns 0 if the buffer is empty. diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index fddb8b156..ede1a1105 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -96,6 +96,7 @@ namespace internal { class DeviceManager; class LogManager; class NotificationQueue; + struct ConformanceTestConfig; class SeqAcqTestMonitor; } // namespace internal } // namespace mmcore @@ -690,6 +691,8 @@ class CMMCore ///@{ void loadMockDeviceAdapter(const char* name, MockDeviceAdapter* implementation) MMCORE_LEGACY_THROW(CMMError); + void setConformanceTestConfig( + const mmcore::internal::ConformanceTestConfig& config); ///@} #endif @@ -729,6 +732,7 @@ class CMMCore std::unique_ptr callback_; std::atomic seqAcqTestMonitor_{nullptr}; + std::unique_ptr conformanceTestConfig_; std::shared_ptr pluginManager_; std::shared_ptr deviceManager_; diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index 6ee42f732..b9b018e06 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -1,6 +1,7 @@ #include #include "DeviceBase.h" +#include "DeviceConformance/ConformanceTestConfig.h" #include "MMCore.h" #include "MMDeviceConstants.h" #include "MockDeviceUtils.h" @@ -24,6 +25,11 @@ bool TestPassed(const nlohmann::json& results, const std::string& testName) { throw std::runtime_error("Test not found: " + testName); } +const mmcore::internal::ConformanceTestConfig shortTimeoutConfig{ + std::chrono::milliseconds(100), + std::chrono::milliseconds(50), +}; + struct ConfigurableAsyncCamera : CCameraBase { std::string name = "ConfigurableAsyncCamera"; unsigned width = 64; @@ -172,6 +178,7 @@ TEST_CASE("Conformant camera passes all conformance tests", ConfigurableAsyncCamera cam; MockAdapterWithDevices adapter{{"cam", &cam}}; CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); adapter.LoadIntoCore(c); c.setCameraDevice("cam"); @@ -190,6 +197,7 @@ TEST_CASE("Missing PrepareForAcq is detected by conformance test", cam.callPrepareForAcq = false; MockAdapterWithDevices adapter{{"cam", &cam}}; CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); adapter.LoadIntoCore(c); c.setCameraDevice("cam"); @@ -204,6 +212,7 @@ TEST_CASE("Missing AcqFinished is detected by conformance test", cam.callAcqFinished = false; MockAdapterWithDevices adapter{{"cam", &cam}}; CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); adapter.LoadIntoCore(c); c.setCameraDevice("cam"); @@ -218,6 +227,7 @@ TEST_CASE("Ignoring InsertImage error return is detected by conformance test", cam.checkInsertImageReturn = false; MockAdapterWithDevices adapter{{"cam", &cam}}; CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); adapter.LoadIntoCore(c); c.setCameraDevice("cam"); From 27916a72f89b776ffee5f06188237767c8ce165c Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Thu, 12 Mar 2026 10:58:45 -0500 Subject: [PATCH 05/17] Refactor conformance tests for all device types (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 244 ++++++------------ MMCore/DeviceConformance/CameraConformance.h | 24 +- .../DeviceConformance/DeviceConformance.cpp | 170 ++++++++++++ MMCore/DeviceConformance/DeviceConformance.h | 54 ++++ MMCore/MMCore.cpp | 29 +-- MMCore/MMCore.h | 2 +- MMCore/meson.build | 1 + MMCore/unittest/CameraConformance-Tests.cpp | 8 +- 8 files changed, 317 insertions(+), 215 deletions(-) create mode 100644 MMCore/DeviceConformance/DeviceConformance.cpp create mode 100644 MMCore/DeviceConformance/DeviceConformance.h diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 931563d67..eb3c60574 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -19,13 +19,6 @@ #include "MMDeviceConstants.h" -#include - -#include -#include -#include -#include -#include #include namespace mmcore { @@ -33,62 +26,16 @@ namespace internal { namespace { -std::string FormatISO8601(std::chrono::system_clock::time_point tp) { - auto time = std::chrono::system_clock::to_time_t(tp); - std::tm tm{}; -#ifdef _WIN32 - gmtime_s(&tm, &time); -#else - gmtime_r(&time, &tm); -#endif - std::ostringstream ss; - ss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); - return ss.str(); -} - -nlohmann::json AssertionToJson(const AssertionResult& a) { - nlohmann::json j; - j["passed"] = a.passed; - j["message"] = a.message; - if (!a.details.empty()) - j["details"] = a.details; - return j; -} - -nlohmann::json TestToJson(const TestResult& t) { - nlohmann::json j; - j["name"] = t.name; - j["passed"] = t.passed; - nlohmann::json assertions = nlohmann::json::array(); - for (const auto& a : t.assertions) - assertions.push_back(AssertionToJson(a)); - j["assertions"] = assertions; - return j; -} - -} // anonymous namespace - -std::string RunCameraConformanceTests( - std::shared_ptr pCam, - std::atomic& seqAcqTestMonitor, - const ConformanceTestConfig& config, - const char* testName, - const std::string& deviceLabel, - const std::string& deviceName, - const std::string& adapterName) { - using namespace std::chrono; - - const auto startTime = system_clock::now(); - const auto startSteady = steady_clock::now(); - - const MM::Device* rawCam = pCam->GetRawPtr(); - const auto testTimeout = config.positiveTimeout; - const auto postErrorDelay = config.negativeTimeout; +struct CameraTestContext { + std::shared_ptr pCam; + std::atomic& seqAcqTestMonitor; + const MM::Device* rawCam; + std::chrono::milliseconds testTimeout; + std::chrono::milliseconds postErrorDelay; - // RAII guard: stop camera first (joining its thread), then clear atomic. struct MonitorGuard { std::atomic& atom; - std::shared_ptr& cam; + std::shared_ptr cam; ~MonitorGuard() { try { DeviceModuleLockGuard g(cam); @@ -99,29 +46,28 @@ std::string RunCameraConformanceTests( } }; - auto startFinite = [&](long numImages, double intervalMs) { + void StartFinite(long numImages, double intervalMs) { DeviceModuleLockGuard guard(pCam); int nRet = pCam->StartSequenceAcquisition(numImages, intervalMs, false); if (nRet != DEVICE_OK) throw CMMError("Camera failed to start finite sequence acquisition"); - }; + } - auto startContinuous = [&](double intervalMs) { + void StartContinuous(double intervalMs) { DeviceModuleLockGuard guard(pCam); int nRet = pCam->StartSequenceAcquisition(intervalMs); if (nRet != DEVICE_OK) - throw CMMError("Camera failed to start continuous sequence acquisition"); - }; + throw CMMError( + "Camera failed to start continuous sequence acquisition"); + } - auto stopCamera = [&]() { + void StopCamera() { DeviceModuleLockGuard guard(pCam); if (pCam->IsCapturing()) pCam->StopSequenceAcquisition(); - }; - - std::vector results; + } - auto testPrepareBeforeInsert = [&]() { + TestResult TestPrepareBeforeInsert() { TestResult result; result.name = "seq-prepare-before-insert"; @@ -129,7 +75,7 @@ std::string RunCameraConformanceTests( seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; - startFinite(5, 0.0); + StartFinite(5, 0.0); monitor.WaitForInsertImageCount(5, testTimeout); if (!monitor.PrepareForAcqCalled()) { @@ -144,10 +90,10 @@ std::string RunCameraConformanceTests( } result.passed = result.assertions.back().passed; - results.push_back(std::move(result)); - }; + return result; + } - auto testFinishedAfterCount = [&]() { + TestResult TestFinishedAfterCount() { TestResult result; result.name = "seq-finished-after-count"; @@ -155,7 +101,7 @@ std::string RunCameraConformanceTests( seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; - startFinite(5, 0.0); + StartFinite(5, 0.0); monitor.WaitForInsertImageCount(5, testTimeout); if (monitor.WaitForAcqFinished(testTimeout)) { @@ -164,23 +110,25 @@ std::string RunCameraConformanceTests( } else { AssertionResult a; a.passed = false; - a.message = "AcqFinished not called after finite sequence (5 frames)"; - stopCamera(); + a.message = + "AcqFinished not called after finite sequence (5 frames)"; + StopCamera(); if (monitor.WaitForAcqFinished(testTimeout)) { a.details.push_back( "AcqFinished was called after stopSequenceAcquisition"); } else { a.details.push_back( - "AcqFinished was not called even after stopSequenceAcquisition"); + "AcqFinished was not called even after " + "stopSequenceAcquisition"); } result.assertions.push_back(std::move(a)); } result.passed = result.assertions.back().passed; - results.push_back(std::move(result)); - }; + return result; + } - auto testFinishedOnError = [&](const char* slug, int errorCode, + TestResult TestFinishedOnError(const char* slug, int errorCode, const char* errorName, bool continuous) { TestResult result; result.name = slug; @@ -191,9 +139,9 @@ std::string RunCameraConformanceTests( MonitorGuard mg{seqAcqTestMonitor, pCam}; if (continuous) - startContinuous(0.0); + StartContinuous(0.0); else - startFinite(1000000, 0.0); + StartFinite(1000000, 0.0); monitor.WaitForInsertImageCount(3, testTimeout); @@ -205,15 +153,17 @@ std::string RunCameraConformanceTests( } else { AssertionResult a; a.passed = false; - a.message = std::string("AcqFinished not called after ") + errorName; + a.message = + std::string("AcqFinished not called after ") + errorName; pass = false; - stopCamera(); + StopCamera(); if (monitor.WaitForAcqFinished(testTimeout)) { a.details.push_back( "AcqFinished was called after stopSequenceAcquisition"); } else { a.details.push_back( - "AcqFinished was not called even after stopSequenceAcquisition"); + "AcqFinished was not called even after " + "stopSequenceAcquisition"); } result.assertions.push_back(std::move(a)); } @@ -231,96 +181,50 @@ std::string RunCameraConformanceTests( } result.passed = pass; - results.push_back(std::move(result)); - }; + return result; + } +}; - auto testFinishedOnErrorFinite = [&]() { - testFinishedOnError("seq-finished-on-error-finite", - DEVICE_ERR, "DEVICE_ERR", false); - }; +} // anonymous namespace - auto testFinishedOnErrorContinuous = [&]() { - testFinishedOnError("seq-finished-on-error-continuous", +std::vector GetCameraConformanceTests( + std::shared_ptr pCam, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config) { + auto ctx = std::make_shared(CameraTestContext{ + pCam, + seqAcqTestMonitor, + pCam->GetRawPtr(), + config.positiveTimeout, + config.negativeTimeout, + }); + + std::vector tests; + + tests.push_back({"seq-prepare-before-insert", [ctx]() { + return ctx->TestPrepareBeforeInsert(); + }}); + tests.push_back({"seq-finished-after-count", [ctx]() { + return ctx->TestFinishedAfterCount(); + }}); + tests.push_back({"seq-finished-on-error-finite", [ctx]() { + return ctx->TestFinishedOnError("seq-finished-on-error-finite", + DEVICE_ERR, "DEVICE_ERR", false); + }}); + tests.push_back({"seq-finished-on-error-continuous", [ctx]() { + return ctx->TestFinishedOnError("seq-finished-on-error-continuous", DEVICE_ERR, "DEVICE_ERR", true); - }; - - auto testFinishedOnOverflowFinite = [&]() { - testFinishedOnError("seq-finished-on-overflow-finite", + }}); + tests.push_back({"seq-finished-on-overflow-finite", [ctx]() { + return ctx->TestFinishedOnError("seq-finished-on-overflow-finite", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); - }; - - auto testFinishedOnOverflowContinuous = [&]() { - testFinishedOnError("seq-finished-on-overflow-continuous", + }}); + tests.push_back({"seq-finished-on-overflow-continuous", [ctx]() { + return ctx->TestFinishedOnError("seq-finished-on-overflow-continuous", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); - }; - - struct TestEntry { - const char* slug; - std::function func; - }; - TestEntry tests[] = { - {"seq-prepare-before-insert", testPrepareBeforeInsert}, - {"seq-finished-after-count", testFinishedAfterCount}, - {"seq-finished-on-error-finite", testFinishedOnErrorFinite}, - {"seq-finished-on-error-continuous", testFinishedOnErrorContinuous}, - {"seq-finished-on-overflow-finite", testFinishedOnOverflowFinite}, - {"seq-finished-on-overflow-continuous", testFinishedOnOverflowContinuous}, - }; - - std::string selectedTest; - if (testName && testName[0] != '\0') - selectedTest = testName; - - if (!selectedTest.empty()) { - bool found = false; - for (const auto& t : tests) { - if (selectedTest == t.slug) { - found = true; - break; - } - } - if (!found) { - throw CMMError("Unknown camera test: " + selectedTest); - } - } - - for (const auto& t : tests) { - if (!selectedTest.empty() && selectedTest != t.slug) - continue; - t.func(); - } - - const auto endSteady = steady_clock::now(); - double durationMs = - duration_cast>(endSteady - startSteady) - .count(); - - int passedCount = 0; - for (const auto& r : results) - if (r.passed) - ++passedCount; - - nlohmann::json testsJson = nlohmann::json::array(); - for (const auto& r : results) - testsJson.push_back(TestToJson(r)); - - nlohmann::json j; - j["version"] = 1; - j["timestamp"] = FormatISO8601(startTime); - j["device"] = { - {"label", deviceLabel}, - {"name", deviceName}, - {"library", adapterName}, - }; - j["deviceType"] = "Camera"; - j["tests"] = testsJson; - j["summary"] = { - {"total", static_cast(results.size())}, - {"passed", passedCount}, - {"durationMs", durationMs}, - }; + }}); - return j.dump(2); + return tests; } } // namespace internal diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h index a8f1d039b..9b26b5101 100644 --- a/MMCore/DeviceConformance/CameraConformance.h +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -13,12 +13,10 @@ #pragma once -#include "ConformanceTestConfig.h" -#include "SeqAcqTestMonitor.h" +#include "DeviceConformance.h" #include #include -#include #include namespace mmcore { @@ -26,26 +24,10 @@ namespace internal { class CameraInstance; -struct AssertionResult { - bool passed; - std::string message; - std::vector details; -}; - -struct TestResult { - std::string name; - bool passed; - std::vector assertions; -}; - -std::string RunCameraConformanceTests( +std::vector GetCameraConformanceTests( std::shared_ptr camera, std::atomic& testMonitor, - const ConformanceTestConfig& config, - const char* testName, - const std::string& deviceLabel, - const std::string& deviceName, - const std::string& adapterName); + const ConformanceTestConfig& config); } // namespace internal } // namespace mmcore diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp new file mode 100644 index 000000000..6fffa1604 --- /dev/null +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -0,0 +1,170 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "DeviceConformance.h" + +#include "CameraConformance.h" +#include "CoreUtils.h" +#include "DeviceManager.h" +#include "Devices/CameraInstance.h" +#include "Devices/DeviceInstance.h" +#include "Error.h" + +#include "MMDeviceConstants.h" + +#include + +#include +#include +#include +#include + +namespace mmcore { +namespace internal { + +namespace { + +std::string FormatISO8601(std::chrono::system_clock::time_point tp) { + auto time = std::chrono::system_clock::to_time_t(tp); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &time); +#else + gmtime_r(&time, &tm); +#endif + std::ostringstream ss; + ss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); + return ss.str(); +} + +nlohmann::json AssertionToJson(const AssertionResult& a) { + nlohmann::json j; + j["passed"] = a.passed; + j["message"] = a.message; + if (!a.details.empty()) + j["details"] = a.details; + return j; +} + +nlohmann::json TestToJson(const TestResult& t) { + nlohmann::json j; + j["name"] = t.name; + j["passed"] = t.passed; + nlohmann::json assertions = nlohmann::json::array(); + for (const auto& a : t.assertions) + assertions.push_back(AssertionToJson(a)); + j["assertions"] = assertions; + return j; +} + +std::string RunConformanceTests( + const std::vector& tests, + const char* testName, + const std::string& deviceLabel, + const std::string& deviceName, + const std::string& adapterName, + const std::string& deviceType) { + using namespace std::chrono; + + std::string selectedTest; + if (testName && testName[0] != '\0') + selectedTest = testName; + + if (!selectedTest.empty()) { + bool found = false; + for (const auto& t : tests) { + if (selectedTest == t.slug) { + found = true; + break; + } + } + if (!found) { + throw CMMError("Unknown conformance test: " + selectedTest); + } + } + + const auto startTime = system_clock::now(); + const auto startSteady = steady_clock::now(); + + std::vector results; + for (const auto& t : tests) { + if (!selectedTest.empty() && selectedTest != t.slug) + continue; + results.push_back(t.func()); + } + + const auto endSteady = steady_clock::now(); + double durationMs = + duration_cast>(endSteady - startSteady) + .count(); + + int passedCount = 0; + for (const auto& r : results) + if (r.passed) + ++passedCount; + + nlohmann::json testsJson = nlohmann::json::array(); + for (const auto& r : results) + testsJson.push_back(TestToJson(r)); + + nlohmann::json j; + j["version"] = 1; + j["timestamp"] = FormatISO8601(startTime); + j["device"] = { + {"label", deviceLabel}, + {"name", deviceName}, + {"library", adapterName}, + }; + j["deviceType"] = deviceType; + j["tests"] = testsJson; + j["summary"] = { + {"total", static_cast(results.size())}, + {"passed", passedCount}, + {"durationMs", durationMs}, + }; + + return j.dump(2); +} + +} // anonymous namespace + +std::string RunDeviceConformanceTests( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config, + const char* testName) { + const auto deviceLabel = device->GetLabel(); + const auto deviceName = device->GetName(); + const auto adapterName = device->GetAdapterModule()->GetName(); + const auto deviceType = device->GetType(); + const auto deviceTypeStr = ToString(deviceType); + + std::vector tests; + if (deviceType == MM::CameraDevice) { + auto pCam = std::static_pointer_cast(device); + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) + throw CMMError( + "Not allowed during sequence acquisition", + MMERR_NotAllowedDuringSequenceAcquisition); + } + tests = GetCameraConformanceTests(pCam, seqAcqTestMonitor, config); + } + + return RunConformanceTests(tests, testName, deviceLabel, deviceName, + adapterName, deviceTypeStr); +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/DeviceConformance.h b/MMCore/DeviceConformance/DeviceConformance.h new file mode 100644 index 000000000..7622a72ab --- /dev/null +++ b/MMCore/DeviceConformance/DeviceConformance.h @@ -0,0 +1,54 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "ConformanceTestConfig.h" +#include "SeqAcqTestMonitor.h" + +#include +#include +#include +#include +#include + +namespace mmcore { +namespace internal { + +class DeviceInstance; + +struct AssertionResult { + bool passed; + std::string message; + std::vector details; +}; + +struct TestResult { + std::string name; + bool passed; + std::vector assertions; +}; + +struct TestEntry { + std::string slug; + std::function func; +}; + +std::string RunDeviceConformanceTests( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config, + const char* testName); + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index 48ec48d6d..1e90879af 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -51,7 +51,7 @@ #include "MMEventCallback.h" #include "NotificationQueue.h" #include "PluginManager.h" -#include "DeviceConformance/CameraConformance.h" +#include "DeviceConformance/DeviceConformance.h" #include "SynchronizedConfiguration.h" #include "DeviceUtils.h" @@ -3090,32 +3090,23 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) }; /** - * Run behavioral tests on a camera device adapter and return a JSON report. + * Run behavioral conformance tests on a device and return a JSON report. * - * The tests exercise the sequence acquisition callback protocol - * (PrepareForAcq, InsertImage, AcqFinished). + * Currently supports camera devices, where the tests exercise the sequence + * acquisition callback protocol (PrepareForAcq, InsertImage, AcqFinished). * - * @param cameraLabel Label of the camera to test (must be loaded and - * initialized, and not currently acquiring). + * @param deviceLabel Label of the device to test (must be loaded and + * initialized). * @param testName Slug name of a single test to run, or null/empty to run * all tests. * @return A JSON string containing the test results. */ -std::string CMMCore::runCameraConformanceTests(const char* cameraLabel, +std::string CMMCore::runDeviceConformanceTests(const char* deviceLabel, const char* testName) MMCORE_LEGACY_THROW(CMMError) { - auto pCam = deviceManager_->GetDeviceOfType(cameraLabel); - { - mmi::DeviceModuleLockGuard guard(pCam); - if (pCam->IsCapturing()) - throw CMMError( - getCoreErrorText(MMERR_NotAllowedDuringSequenceAcquisition), - MMERR_NotAllowedDuringSequenceAcquisition); - } - - return mmi::RunCameraConformanceTests(pCam, seqAcqTestMonitor_, - *conformanceTestConfig_, testName, - cameraLabel, pCam->GetName(), pCam->GetAdapterModule()->GetName()); + auto device = deviceManager_->GetDevice(deviceLabel); + return mmi::RunDeviceConformanceTests(device, seqAcqTestMonitor_, + *conformanceTestConfig_, testName); } /** diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index ede1a1105..89d290fc1 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -682,7 +682,7 @@ class CMMCore /** \name Device conformance testing. */ ///@{ - std::string runCameraConformanceTests(const char* cameraLabel, + std::string runDeviceConformanceTests(const char* deviceLabel, const char* testName = nullptr) MMCORE_LEGACY_THROW(CMMError); ///@} diff --git a/MMCore/meson.build b/MMCore/meson.build index efe02f6dc..c472887e8 100644 --- a/MMCore/meson.build +++ b/MMCore/meson.build @@ -65,6 +65,7 @@ mmcore_sources = files( 'MMCore.cpp', 'PluginManager.cpp', 'DeviceConformance/CameraConformance.cpp', + 'DeviceConformance/DeviceConformance.cpp', 'DeviceConformance/SeqAcqTestMonitor.cpp', 'Semaphore.cpp', 'Task.cpp', diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index b9b018e06..a84e52571 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -182,7 +182,7 @@ TEST_CASE("Conformant camera passes all conformance tests", adapter.LoadIntoCore(c); c.setCameraDevice("cam"); - auto results = nlohmann::json::parse(c.runCameraConformanceTests("cam")); + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); for (const auto& test : results["tests"]) { INFO("Test: " << test["name"].get()); CHECK(test["passed"].get()); @@ -202,7 +202,7 @@ TEST_CASE("Missing PrepareForAcq is detected by conformance test", c.setCameraDevice("cam"); auto results = nlohmann::json::parse( - c.runCameraConformanceTests("cam", "seq-prepare-before-insert")); + c.runDeviceConformanceTests("cam", "seq-prepare-before-insert")); CHECK_FALSE(TestPassed(results, "seq-prepare-before-insert")); } @@ -217,7 +217,7 @@ TEST_CASE("Missing AcqFinished is detected by conformance test", c.setCameraDevice("cam"); auto results = nlohmann::json::parse( - c.runCameraConformanceTests("cam", "seq-finished-after-count")); + c.runDeviceConformanceTests("cam", "seq-finished-after-count")); CHECK_FALSE(TestPassed(results, "seq-finished-after-count")); } @@ -232,6 +232,6 @@ TEST_CASE("Ignoring InsertImage error return is detected by conformance test", c.setCameraDevice("cam"); auto results = nlohmann::json::parse( - c.runCameraConformanceTests("cam", "seq-finished-on-error-finite")); + c.runDeviceConformanceTests("cam", "seq-finished-on-error-finite")); CHECK_FALSE(TestPassed(results, "seq-finished-on-error-finite")); } From a18496a6f4b7548904e1f836c70bf61db5fd243d Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 13 Mar 2026 13:10:18 -0500 Subject: [PATCH 06/17] Conformance test warning status and skipping Add statuses: wanring and skipped; let tests depend on earlier tests having passes (otherwise skip). (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 75 ++++++++++------ .../DeviceConformance/DeviceConformance.cpp | 88 ++++++++++++++++--- MMCore/DeviceConformance/DeviceConformance.h | 8 +- MMCore/unittest/CameraConformance-Tests.cpp | 49 +++++++++-- 4 files changed, 173 insertions(+), 47 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index eb3c60574..29337b289 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -75,21 +75,29 @@ struct CameraTestContext { seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; - StartFinite(5, 0.0); + try { + StartFinite(5, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } monitor.WaitForInsertImageCount(5, testTimeout); if (!monitor.PrepareForAcqCalled()) { result.assertions.push_back( - {false, "PrepareForAcq was not called", {}}); + {AssertionStatus::Fail, "PrepareForAcq was not called", {}}); } else if (!monitor.PrepareBeforeFirstInsert()) { result.assertions.push_back( - {false, "PrepareForAcq was called after InsertImage", {}}); + {AssertionStatus::Fail, + "PrepareForAcq was called after InsertImage", {}}); } else { result.assertions.push_back( - {true, "PrepareForAcq called before first InsertImage", {}}); + {AssertionStatus::Pass, + "PrepareForAcq called before first InsertImage", {}}); } - result.passed = result.assertions.back().passed; return result; } @@ -101,15 +109,23 @@ struct CameraTestContext { seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; - StartFinite(5, 0.0); + try { + StartFinite(5, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } monitor.WaitForInsertImageCount(5, testTimeout); if (monitor.WaitForAcqFinished(testTimeout)) { result.assertions.push_back( - {true, "AcqFinished called after finite sequence completed", {}}); + {AssertionStatus::Pass, + "AcqFinished called after finite sequence completed", {}}); } else { AssertionResult a; - a.passed = false; + a.status = AssertionStatus::Fail; a.message = "AcqFinished not called after finite sequence (5 frames)"; StopCamera(); @@ -124,7 +140,6 @@ struct CameraTestContext { result.assertions.push_back(std::move(a)); } - result.passed = result.assertions.back().passed; return result; } @@ -138,24 +153,29 @@ struct CameraTestContext { seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; - if (continuous) - StartContinuous(0.0); - else - StartFinite(1000000, 0.0); + try { + if (continuous) + StartContinuous(0.0); + else + StartFinite(1000000, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } monitor.WaitForInsertImageCount(3, testTimeout); - bool pass = true; - if (monitor.WaitForAcqFinished(testTimeout)) { result.assertions.push_back( - {true, std::string("AcqFinished called after ") + errorName, {}}); + {AssertionStatus::Pass, + std::string("AcqFinished called after ") + errorName, {}}); } else { AssertionResult a; - a.passed = false; + a.status = AssertionStatus::Fail; a.message = std::string("AcqFinished not called after ") + errorName; - pass = false; StopCamera(); if (monitor.WaitForAcqFinished(testTimeout)) { a.details.push_back( @@ -172,15 +192,14 @@ struct CameraTestContext { int afterError = monitor.InsertImageCountAfterError(); if (afterError > 1) { result.assertions.push_back( - {false, std::to_string(afterError - 1) + + {AssertionStatus::Fail, std::to_string(afterError - 1) + " InsertImage call(s) after error return", {}}); - pass = false; } else { result.assertions.push_back( - {true, "No further InsertImage calls after error", {}}); + {AssertionStatus::Pass, + "No further InsertImage calls after error", {}}); } - result.passed = pass; return result; } }; @@ -203,26 +222,26 @@ std::vector GetCameraConformanceTests( tests.push_back({"seq-prepare-before-insert", [ctx]() { return ctx->TestPrepareBeforeInsert(); - }}); + }, {}}); tests.push_back({"seq-finished-after-count", [ctx]() { return ctx->TestFinishedAfterCount(); - }}); + }, {"seq-prepare-before-insert"}}); tests.push_back({"seq-finished-on-error-finite", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-error-finite", DEVICE_ERR, "DEVICE_ERR", false); - }}); + }, {"seq-prepare-before-insert"}}); tests.push_back({"seq-finished-on-error-continuous", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-error-continuous", DEVICE_ERR, "DEVICE_ERR", true); - }}); + }, {"seq-prepare-before-insert"}}); tests.push_back({"seq-finished-on-overflow-finite", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-overflow-finite", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); - }}); + }, {"seq-prepare-before-insert"}}); tests.push_back({"seq-finished-on-overflow-continuous", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-overflow-continuous", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); - }}); + }, {"seq-prepare-before-insert"}}); return tests; } diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp index 6fffa1604..60fb9da94 100644 --- a/MMCore/DeviceConformance/DeviceConformance.cpp +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace mmcore { namespace internal { @@ -47,19 +48,49 @@ std::string FormatISO8601(std::chrono::system_clock::time_point tp) { return ss.str(); } +const char* AssertionStatusToString(AssertionStatus s) { + switch (s) { + case AssertionStatus::Pass: return "pass"; + case AssertionStatus::Fail: return "fail"; + case AssertionStatus::Warning: return "warning"; + } + return "unknown"; +} + +const char* TestStatusToString(TestStatus s) { + switch (s) { + case TestStatus::Pass: return "pass"; + case TestStatus::Fail: return "fail"; + case TestStatus::Warning: return "warning"; + case TestStatus::Skipped: return "skipped"; + } + return "unknown"; +} + +TestStatus DeriveTestStatus(const std::vector& assertions) { + bool anyWarning = false; + for (const auto& a : assertions) { + if (a.status == AssertionStatus::Fail) + return TestStatus::Fail; + if (a.status == AssertionStatus::Warning) + anyWarning = true; + } + return anyWarning ? TestStatus::Warning : TestStatus::Pass; +} + nlohmann::json AssertionToJson(const AssertionResult& a) { nlohmann::json j; - j["passed"] = a.passed; + j["status"] = AssertionStatusToString(a.status); j["message"] = a.message; if (!a.details.empty()) j["details"] = a.details; return j; } -nlohmann::json TestToJson(const TestResult& t) { +nlohmann::json TestToJson(const TestResult& t, TestStatus status) { nlohmann::json j; j["name"] = t.name; - j["passed"] = t.passed; + j["status"] = TestStatusToString(status); nlohmann::json assertions = nlohmann::json::array(); for (const auto& a : t.assertions) assertions.push_back(AssertionToJson(a)); @@ -97,10 +128,37 @@ std::string RunConformanceTests( const auto startSteady = steady_clock::now(); std::vector results; + std::vector statuses; + std::unordered_map completedStatuses; + for (const auto& t : tests) { if (!selectedTest.empty() && selectedTest != t.slug) continue; - results.push_back(t.func()); + + bool skip = false; + if (selectedTest.empty()) { + for (const auto& dep : t.dependsOn) { + auto it = completedStatuses.find(dep); + if (it == completedStatuses.end() || + it->second != TestStatus::Pass) { + skip = true; + break; + } + } + } + + TestStatus status; + if (skip) { + TestResult r; + r.name = t.slug; + results.push_back(std::move(r)); + status = TestStatus::Skipped; + } else { + results.push_back(t.func()); + status = DeriveTestStatus(results.back().assertions); + } + statuses.push_back(status); + completedStatuses[t.slug] = status; } const auto endSteady = steady_clock::now(); @@ -108,17 +166,22 @@ std::string RunConformanceTests( duration_cast>(endSteady - startSteady) .count(); - int passedCount = 0; - for (const auto& r : results) - if (r.passed) - ++passedCount; + int passedCount = 0, failedCount = 0, warningCount = 0, skippedCount = 0; + for (auto s : statuses) { + switch (s) { + case TestStatus::Pass: ++passedCount; break; + case TestStatus::Fail: ++failedCount; break; + case TestStatus::Warning: ++warningCount; break; + case TestStatus::Skipped: ++skippedCount; break; + } + } nlohmann::json testsJson = nlohmann::json::array(); - for (const auto& r : results) - testsJson.push_back(TestToJson(r)); + for (size_t i = 0; i < results.size(); ++i) + testsJson.push_back(TestToJson(results[i], statuses[i])); nlohmann::json j; - j["version"] = 1; + j["version"] = 2; j["timestamp"] = FormatISO8601(startTime); j["device"] = { {"label", deviceLabel}, @@ -130,6 +193,9 @@ std::string RunConformanceTests( j["summary"] = { {"total", static_cast(results.size())}, {"passed", passedCount}, + {"failed", failedCount}, + {"warnings", warningCount}, + {"skipped", skippedCount}, {"durationMs", durationMs}, }; diff --git a/MMCore/DeviceConformance/DeviceConformance.h b/MMCore/DeviceConformance/DeviceConformance.h index 7622a72ab..56914ef4c 100644 --- a/MMCore/DeviceConformance/DeviceConformance.h +++ b/MMCore/DeviceConformance/DeviceConformance.h @@ -27,21 +27,25 @@ namespace internal { class DeviceInstance; +enum class AssertionStatus { Pass, Fail, Warning }; + struct AssertionResult { - bool passed; + AssertionStatus status; std::string message; std::vector details; }; +enum class TestStatus { Pass, Fail, Warning, Skipped }; + struct TestResult { std::string name; - bool passed; std::vector assertions; }; struct TestEntry { std::string slug; std::function func; + std::vector dependsOn; }; std::string RunDeviceConformanceTests( diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index a84e52571..212772168 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -17,10 +17,11 @@ namespace { -bool TestPassed(const nlohmann::json& results, const std::string& testName) { +std::string GetTestStatus(const nlohmann::json& results, + const std::string& testName) { for (const auto& t : results["tests"]) { if (t["name"] == testName) - return t["passed"].get(); + return t["status"].get(); } throw std::runtime_error("Test not found: " + testName); } @@ -43,6 +44,7 @@ struct ConfigurableAsyncCamera : CCameraBase { bool callPrepareForAcq = true; bool callAcqFinished = true; bool checkInsertImageReturn = true; + bool failStartSequenceAcq = false; int Initialize() override { return DEVICE_OK; } int Shutdown() override { return DEVICE_OK; } @@ -85,6 +87,8 @@ struct ConfigurableAsyncCamera : CCameraBase { } int StartSequenceAcquisition(long numImages, double, bool) override { + if (failStartSequenceAcq) + return DEVICE_ERR; // Thread may be left over from previous (unstopped) run if (thread_.joinable()) thread_.join(); @@ -185,7 +189,7 @@ TEST_CASE("Conformant camera passes all conformance tests", auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); for (const auto& test : results["tests"]) { INFO("Test: " << test["name"].get()); - CHECK(test["passed"].get()); + CHECK(test["status"].get() == "pass"); } CHECK(results["summary"]["passed"].get() == results["summary"]["total"].get()); @@ -203,7 +207,7 @@ TEST_CASE("Missing PrepareForAcq is detected by conformance test", auto results = nlohmann::json::parse( c.runDeviceConformanceTests("cam", "seq-prepare-before-insert")); - CHECK_FALSE(TestPassed(results, "seq-prepare-before-insert")); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "fail"); } TEST_CASE("Missing AcqFinished is detected by conformance test", @@ -218,7 +222,7 @@ TEST_CASE("Missing AcqFinished is detected by conformance test", auto results = nlohmann::json::parse( c.runDeviceConformanceTests("cam", "seq-finished-after-count")); - CHECK_FALSE(TestPassed(results, "seq-finished-after-count")); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "fail"); } TEST_CASE("Ignoring InsertImage error return is detected by conformance test", @@ -233,5 +237,38 @@ TEST_CASE("Ignoring InsertImage error return is detected by conformance test", auto results = nlohmann::json::parse( c.runDeviceConformanceTests("cam", "seq-finished-on-error-finite")); - CHECK_FALSE(TestPassed(results, "seq-finished-on-error-finite")); + CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "fail"); +} + +TEST_CASE("Camera that fails to start seq acq produces warning", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.failStartSequenceAcq = true; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-prepare-before-insert")); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "warning"); +} + +TEST_CASE("Dependent tests are skipped when dependency warns", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.failStartSequenceAcq = true; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "warning"); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "skipped"); + CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "skipped"); + CHECK(results["summary"]["warnings"].get() == 1); + CHECK(results["summary"]["skipped"].get() == 5); } From b7de85a052924d2770af3351ff6ad2cc11851e89 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 13 Mar 2026 13:33:55 -0500 Subject: [PATCH 07/17] Add basic sequence acquisition conformance test Fail early if the bare minimum doesn't work. (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 45 ++++++++++++++++--- MMCore/unittest/CameraConformance-Tests.cpp | 9 ++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 29337b289..5e90d0db8 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -67,6 +67,36 @@ struct CameraTestContext { pCam->StopSequenceAcquisition(); } + TestResult TestSeqBasic() { + TestResult result; + result.name = "seq-basic"; + + SeqAcqTestMonitor monitor(rawCam); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + try { + StartFinite(1, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + + if (monitor.WaitForInsertImageCount(1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + "Camera produced 1 image via sequence acquisition", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "No image arrived from sequence acquisition", {}}); + } + + return result; + } + TestResult TestPrepareBeforeInsert() { TestResult result; result.name = "seq-prepare-before-insert"; @@ -220,28 +250,31 @@ std::vector GetCameraConformanceTests( std::vector tests; + tests.push_back({"seq-basic", [ctx]() { + return ctx->TestSeqBasic(); + }, {}}); tests.push_back({"seq-prepare-before-insert", [ctx]() { return ctx->TestPrepareBeforeInsert(); - }, {}}); + }, {"seq-basic"}}); tests.push_back({"seq-finished-after-count", [ctx]() { return ctx->TestFinishedAfterCount(); - }, {"seq-prepare-before-insert"}}); + }, {"seq-basic"}}); tests.push_back({"seq-finished-on-error-finite", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-error-finite", DEVICE_ERR, "DEVICE_ERR", false); - }, {"seq-prepare-before-insert"}}); + }, {"seq-basic"}}); tests.push_back({"seq-finished-on-error-continuous", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-error-continuous", DEVICE_ERR, "DEVICE_ERR", true); - }, {"seq-prepare-before-insert"}}); + }, {"seq-basic"}}); tests.push_back({"seq-finished-on-overflow-finite", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-overflow-finite", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); - }, {"seq-prepare-before-insert"}}); + }, {"seq-basic"}}); tests.push_back({"seq-finished-on-overflow-continuous", [ctx]() { return ctx->TestFinishedOnError("seq-finished-on-overflow-continuous", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); - }, {"seq-prepare-before-insert"}}); + }, {"seq-basic"}}); return tests; } diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index 212772168..ca567c5d5 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -251,8 +251,8 @@ TEST_CASE("Camera that fails to start seq acq produces warning", c.setCameraDevice("cam"); auto results = nlohmann::json::parse( - c.runDeviceConformanceTests("cam", "seq-prepare-before-insert")); - CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "warning"); + c.runDeviceConformanceTests("cam", "seq-basic")); + CHECK(GetTestStatus(results, "seq-basic") == "warning"); } TEST_CASE("Dependent tests are skipped when dependency warns", @@ -266,9 +266,10 @@ TEST_CASE("Dependent tests are skipped when dependency warns", c.setCameraDevice("cam"); auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); - CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "warning"); + CHECK(GetTestStatus(results, "seq-basic") == "warning"); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-after-count") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "skipped"); CHECK(results["summary"]["warnings"].get() == 1); - CHECK(results["summary"]["skipped"].get() == 5); + CHECK(results["summary"]["skipped"].get() == 6); } From 6ff236ae6a83eaabfb068fc5686f8651a0ac73d5 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 13 Mar 2026 13:45:15 -0500 Subject: [PATCH 08/17] Camera confirmance: don't skip Stop calls For mid-test stopping (e.g., to see if the camera incorrectly calls AcqFinished() only upon StopSequenceAcquisition() call), we should call regardless of IsCapturing() status. For test cleanup (MonitorGuard), we should call StopSequenceAcquisition() in all cases. (Assisted by Claude Code; any errors are mine.) --- MMCore/DeviceConformance/CameraConformance.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 5e90d0db8..899a108ff 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -39,8 +39,7 @@ struct CameraTestContext { ~MonitorGuard() { try { DeviceModuleLockGuard g(cam); - if (cam->IsCapturing()) - cam->StopSequenceAcquisition(); + cam->StopSequenceAcquisition(); } catch (...) {} atom.store(nullptr, std::memory_order_release); } @@ -63,8 +62,7 @@ struct CameraTestContext { void StopCamera() { DeviceModuleLockGuard guard(pCam); - if (pCam->IsCapturing()) - pCam->StopSequenceAcquisition(); + pCam->StopSequenceAcquisition(); } TestResult TestSeqBasic() { From ee0821771975bc18fd1f877af63cfa8f559e0637 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 13 Mar 2026 14:25:04 -0500 Subject: [PATCH 09/17] Capture properties and settings in test results Probably useful for interpreting results, because the tests run at (or starting at) the current settings. (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 85 +++++++++++++++++++ MMCore/DeviceConformance/CameraConformance.h | 5 ++ .../DeviceConformance/DeviceConformance.cpp | 26 +++++- MMCore/DeviceConformance/DeviceConformance.h | 5 ++ MMCore/unittest/CameraConformance-Tests.cpp | 28 ++++++ 5 files changed, 146 insertions(+), 3 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 899a108ff..242072ca5 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -19,11 +19,96 @@ #include "MMDeviceConstants.h" +#include + #include +#include namespace mmcore { namespace internal { +nlohmann::json CollectCameraState( + std::shared_ptr camera) { + nlohmann::json s = nlohmann::json::object(); + DeviceModuleLockGuard guard(camera); + + try { s["exposure"] = camera->GetExposure(); } + catch (...) { s["exposure"] = nullptr; } + + try { s["binning"] = camera->GetBinning(); } + catch (...) { s["binning"] = nullptr; } + + try { + unsigned x, y, w, h; + camera->GetROI(x, y, w, h); + s["roi"] = {{"x", x}, {"y", y}, {"width", w}, {"height", h}}; + } catch (...) { s["roi"] = nullptr; } + + try { s["imageWidth"] = camera->GetImageWidth(); } + catch (...) { s["imageWidth"] = nullptr; } + + try { s["imageHeight"] = camera->GetImageHeight(); } + catch (...) { s["imageHeight"] = nullptr; } + + try { s["imageBytesPerPixel"] = camera->GetImageBytesPerPixel(); } + catch (...) { s["imageBytesPerPixel"] = nullptr; } + + try { s["bitDepth"] = camera->GetBitDepth(); } + catch (...) { s["bitDepth"] = nullptr; } + + try { s["numberOfComponents"] = camera->GetNumberOfComponents(); } + catch (...) { s["numberOfComponents"] = nullptr; } + + try { + unsigned nCh = camera->GetNumberOfChannels(); + nlohmann::json channels = nlohmann::json::array(); + for (unsigned i = 0; i < nCh; ++i) + channels.push_back(camera->GetChannelName(i)); + s["channels"] = channels; + } catch (...) { s["channels"] = nullptr; } + + try { s["imageBufferSize"] = camera->GetImageBufferSize(); } + catch (...) { s["imageBufferSize"] = nullptr; } + + try { s["multiROISupported"] = camera->SupportsMultiROI(); } + catch (...) { s["multiROISupported"] = nullptr; } + + try { s["multiROIEnabled"] = camera->IsMultiROISet(); } + catch (...) { s["multiROIEnabled"] = nullptr; } + + try { + if (camera->IsMultiROISet()) { + unsigned count = 0; + camera->GetMultiROI(nullptr, nullptr, nullptr, nullptr, &count); + std::vector xs(count), ys(count), ws(count), hs(count); + camera->GetMultiROI(xs.data(), ys.data(), ws.data(), hs.data(), + &count); + nlohmann::json rois = nlohmann::json::array(); + for (unsigned i = 0; i < count; ++i) + rois.push_back( + {{"x", xs[i]}, {"y", ys[i]}, + {"width", ws[i]}, {"height", hs[i]}}); + s["multiROIs"] = rois; + } + } catch (...) { s["multiROIs"] = nullptr; } + + bool isSeq = false; + try { + camera->IsExposureSequenceable(isSeq); + s["exposureSequenceable"] = isSeq; + } catch (...) { s["exposureSequenceable"] = nullptr; } + + if (isSeq) { + try { + long maxLen = 0; + camera->GetExposureSequenceMaxLength(maxLen); + s["exposureSequenceMaxLength"] = maxLen; + } catch (...) { s["exposureSequenceMaxLength"] = nullptr; } + } + + return s; +} + namespace { struct CameraTestContext { diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h index 9b26b5101..911e677ed 100644 --- a/MMCore/DeviceConformance/CameraConformance.h +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -19,11 +19,16 @@ #include #include +#include + namespace mmcore { namespace internal { class CameraInstance; +nlohmann::json CollectCameraState( + std::shared_ptr camera); + std::vector GetCameraConformanceTests( std::shared_ptr camera, std::atomic& testMonitor, diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp index 60fb9da94..bda5080ac 100644 --- a/MMCore/DeviceConformance/DeviceConformance.cpp +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -104,7 +104,8 @@ std::string RunConformanceTests( const std::string& deviceLabel, const std::string& deviceName, const std::string& adapterName, - const std::string& deviceType) { + const std::string& deviceType, + const nlohmann::json& deviceState) { using namespace std::chrono; std::string selectedTest; @@ -181,7 +182,7 @@ std::string RunConformanceTests( testsJson.push_back(TestToJson(results[i], statuses[i])); nlohmann::json j; - j["version"] = 2; + j["version"] = 3; j["timestamp"] = FormatISO8601(startTime); j["device"] = { {"label", deviceLabel}, @@ -189,6 +190,7 @@ std::string RunConformanceTests( {"library", adapterName}, }; j["deviceType"] = deviceType; + j["deviceState"] = deviceState; j["tests"] = testsJson; j["summary"] = { {"total", static_cast(results.size())}, @@ -204,6 +206,20 @@ std::string RunConformanceTests( } // anonymous namespace +nlohmann::json CollectDeviceProperties( + std::shared_ptr device) { + nlohmann::json props = nlohmann::json::object(); + DeviceModuleLockGuard guard(device); + for (const auto& name : device->GetPropertyNames()) { + try { + props[name] = device->GetProperty(name); + } catch (...) { + props[name] = nullptr; + } + } + return props; +} + std::string RunDeviceConformanceTests( std::shared_ptr device, std::atomic& seqAcqTestMonitor, @@ -215,6 +231,9 @@ std::string RunDeviceConformanceTests( const auto deviceType = device->GetType(); const auto deviceTypeStr = ToString(deviceType); + nlohmann::json deviceState; + deviceState["properties"] = CollectDeviceProperties(device); + std::vector tests; if (deviceType == MM::CameraDevice) { auto pCam = std::static_pointer_cast(device); @@ -225,11 +244,12 @@ std::string RunDeviceConformanceTests( "Not allowed during sequence acquisition", MMERR_NotAllowedDuringSequenceAcquisition); } + deviceState["settings"] = CollectCameraState(pCam); tests = GetCameraConformanceTests(pCam, seqAcqTestMonitor, config); } return RunConformanceTests(tests, testName, deviceLabel, deviceName, - adapterName, deviceTypeStr); + adapterName, deviceTypeStr, deviceState); } } // namespace internal diff --git a/MMCore/DeviceConformance/DeviceConformance.h b/MMCore/DeviceConformance/DeviceConformance.h index 56914ef4c..5e884d439 100644 --- a/MMCore/DeviceConformance/DeviceConformance.h +++ b/MMCore/DeviceConformance/DeviceConformance.h @@ -22,6 +22,8 @@ #include #include +#include + namespace mmcore { namespace internal { @@ -48,6 +50,9 @@ struct TestEntry { std::vector dependsOn; }; +nlohmann::json CollectDeviceProperties( + std::shared_ptr device); + std::string RunDeviceConformanceTests( std::shared_ptr device, std::atomic& seqAcqTestMonitor, diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index ca567c5d5..96cf5accb 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -273,3 +273,31 @@ TEST_CASE("Dependent tests are skipped when dependency warns", CHECK(results["summary"]["warnings"].get() == 1); CHECK(results["summary"]["skipped"].get() == 6); } + +TEST_CASE("deviceState contains camera settings", "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); + CHECK(results["version"].get() == 3); + REQUIRE(results["deviceState"].is_object()); + + const auto& state = results["deviceState"]; + REQUIRE(state["properties"].is_object()); + REQUIRE(state["settings"].is_object()); + + const auto& settings = state["settings"]; + CHECK(settings["exposure"].get() == 10.0); + CHECK(settings["binning"].get() == 1); + CHECK(settings["imageWidth"].get() == 64); + CHECK(settings["imageHeight"].get() == 64); + CHECK(settings["bitDepth"].get() == 8); + CHECK(settings["roi"] == + nlohmann::json({{"x", 0}, {"y", 0}, {"width", 64}, {"height", 64}})); + CHECK(settings["exposureSequenceable"].get() == false); + CHECK(settings["multiROISupported"].get() == false); +} From f987a21297310b1822a7b4f647cebddf77744fc0 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 17 Mar 2026 09:36:22 -0500 Subject: [PATCH 10/17] Use log-based tracking for seq acq conf tests Simplify, prevent missed events, and do more accurate checks. (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 70 +++++++++++++----- .../DeviceConformance/SeqAcqTestMonitor.cpp | 72 +++++++------------ MMCore/DeviceConformance/SeqAcqTestMonitor.h | 29 ++++---- 3 files changed, 93 insertions(+), 78 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 242072ca5..d71552ad7 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -111,12 +111,43 @@ nlohmann::json CollectCameraState( namespace { +bool HasEvent(const std::vector& log, SeqAcqEvent event) { + for (const auto& e : log) + if (e.event == event) + return true; + return false; +} + +bool PrepareBeforeFirstInsert(const std::vector& log) { + for (const auto& e : log) { + if (e.event == SeqAcqEvent::PrepareForAcq) + return true; + if (e.event == SeqAcqEvent::InsertImage) + return false; + } + return false; +} + +int CountInsertsAfterError(const std::vector& log) { + bool seenError = false; + int count = 0; + for (const auto& e : log) { + if (e.event != SeqAcqEvent::InsertImage) + continue; + if (seenError) + ++count; + else if (e.returnCode != DEVICE_OK) + seenError = true; + } + return count; +} + struct CameraTestContext { std::shared_ptr pCam; std::atomic& seqAcqTestMonitor; const MM::Device* rawCam; std::chrono::milliseconds testTimeout; - std::chrono::milliseconds postErrorDelay; + std::chrono::milliseconds negativeTimeout; struct MonitorGuard { std::atomic& atom; @@ -167,7 +198,7 @@ struct CameraTestContext { return result; } - if (monitor.WaitForInsertImageCount(1, testTimeout)) { + if (monitor.WaitForEvent(SeqAcqEvent::InsertImage, 1, testTimeout)) { result.assertions.push_back( {AssertionStatus::Pass, "Camera produced 1 image via sequence acquisition", {}}); @@ -196,12 +227,13 @@ struct CameraTestContext { "Camera failed to start sequence acquisition", {}}); return result; } - monitor.WaitForInsertImageCount(5, testTimeout); + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 5, testTimeout); - if (!monitor.PrepareForAcqCalled()) { + auto log = monitor.GetLog(); + if (!HasEvent(log, SeqAcqEvent::PrepareForAcq)) { result.assertions.push_back( {AssertionStatus::Fail, "PrepareForAcq was not called", {}}); - } else if (!monitor.PrepareBeforeFirstInsert()) { + } else if (!PrepareBeforeFirstInsert(log)) { result.assertions.push_back( {AssertionStatus::Fail, "PrepareForAcq was called after InsertImage", {}}); @@ -230,9 +262,9 @@ struct CameraTestContext { "Camera failed to start sequence acquisition", {}}); return result; } - monitor.WaitForInsertImageCount(5, testTimeout); + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 5, testTimeout); - if (monitor.WaitForAcqFinished(testTimeout)) { + if (monitor.WaitForEvent(SeqAcqEvent::AcqFinished, 1, testTimeout)) { result.assertions.push_back( {AssertionStatus::Pass, "AcqFinished called after finite sequence completed", {}}); @@ -242,7 +274,8 @@ struct CameraTestContext { a.message = "AcqFinished not called after finite sequence (5 frames)"; StopCamera(); - if (monitor.WaitForAcqFinished(testTimeout)) { + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { a.details.push_back( "AcqFinished was called after stopSequenceAcquisition"); } else { @@ -278,9 +311,10 @@ struct CameraTestContext { return result; } - monitor.WaitForInsertImageCount(3, testTimeout); + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 4, testTimeout); - if (monitor.WaitForAcqFinished(testTimeout)) { + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { result.assertions.push_back( {AssertionStatus::Pass, std::string("AcqFinished called after ") + errorName, {}}); @@ -290,7 +324,8 @@ struct CameraTestContext { a.message = std::string("AcqFinished not called after ") + errorName; StopCamera(); - if (monitor.WaitForAcqFinished(testTimeout)) { + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { a.details.push_back( "AcqFinished was called after stopSequenceAcquisition"); } else { @@ -301,11 +336,12 @@ struct CameraTestContext { result.assertions.push_back(std::move(a)); } - std::this_thread::sleep_for(postErrorDelay); - int afterError = monitor.InsertImageCountAfterError(); - if (afterError > 1) { + std::this_thread::sleep_for(negativeTimeout); + auto log = monitor.GetLog(); + int afterError = CountInsertsAfterError(log); + if (afterError > 0) { result.assertions.push_back( - {AssertionStatus::Fail, std::to_string(afterError - 1) + + {AssertionStatus::Fail, std::to_string(afterError) + " InsertImage call(s) after error return", {}}); } else { result.assertions.push_back( @@ -328,8 +364,8 @@ std::vector GetCameraConformanceTests( seqAcqTestMonitor, pCam->GetRawPtr(), config.positiveTimeout, - config.negativeTimeout, - }); + config.negativeTimeout}); + std::vector tests; diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp index 42f3587b7..28f186304 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp @@ -25,73 +25,51 @@ void SeqAcqTestMonitor::SetErrorInjection(int errorCode, void SeqAcqTestMonitor::OnPrepareForAcq() { std::lock_guard lock(mutex_); - prepareForAcqCalled_ = true; - if (insertImageCount_ == 0) - prepareBeforeFirstInsert_ = true; + log_.push_back({SeqAcqEvent::PrepareForAcq, DEVICE_OK, + std::chrono::steady_clock::now()}); + cv_.notify_all(); } int SeqAcqTestMonitor::OnInsertImage() { std::lock_guard lock(mutex_); - if (errorInjected_) { - ++insertImageCountAfterError_; - cv_.notify_all(); - return injectErrorCode_; - } + int retCode = DEVICE_OK; if (injectErrorCode_ != DEVICE_OK && - insertImageCount_ >= injectAfterCount_) { + successfulInsertCount_ >= injectAfterCount_) { errorInjected_ = true; - ++insertImageCountAfterError_; - cv_.notify_all(); - return injectErrorCode_; } - ++insertImageCount_; + if (errorInjected_) { + retCode = injectErrorCode_; + } else { + ++successfulInsertCount_; + } + log_.push_back({SeqAcqEvent::InsertImage, retCode, + std::chrono::steady_clock::now()}); cv_.notify_all(); - return DEVICE_OK; + return retCode; } void SeqAcqTestMonitor::OnAcqFinished() { std::lock_guard lock(mutex_); - acqFinishedCalled_ = true; + log_.push_back({SeqAcqEvent::AcqFinished, DEVICE_OK, + std::chrono::steady_clock::now()}); cv_.notify_all(); } -bool SeqAcqTestMonitor::WaitForInsertImageCount(int n, +bool SeqAcqTestMonitor::WaitForEvent(SeqAcqEvent event, int count, std::chrono::milliseconds timeout) { std::unique_lock lock(mutex_); - return cv_.wait_for(lock, timeout, - [&] { return insertImageCount_ >= n || errorInjected_; }); -} - -bool SeqAcqTestMonitor::WaitForAcqFinished( - std::chrono::milliseconds timeout) { - std::unique_lock lock(mutex_); - return cv_.wait_for(lock, timeout, - [&] { return acqFinishedCalled_; }); -} - -bool SeqAcqTestMonitor::PrepareForAcqCalled() const { - std::lock_guard lock(mutex_); - return prepareForAcqCalled_; -} - -int SeqAcqTestMonitor::InsertImageCount() const { - std::lock_guard lock(mutex_); - return insertImageCount_; -} - -bool SeqAcqTestMonitor::AcqFinishedCalled() const { - std::lock_guard lock(mutex_); - return acqFinishedCalled_; -} - -bool SeqAcqTestMonitor::PrepareBeforeFirstInsert() const { - std::lock_guard lock(mutex_); - return prepareBeforeFirstInsert_; + return cv_.wait_for(lock, timeout, [&] { + int n = 0; + for (const auto& entry : log_) + if (entry.event == event) + ++n; + return n >= count; + }); } -int SeqAcqTestMonitor::InsertImageCountAfterError() const { +std::vector SeqAcqTestMonitor::GetLog() const { std::lock_guard lock(mutex_); - return insertImageCountAfterError_; + return log_; } } // namespace internal diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.h b/MMCore/DeviceConformance/SeqAcqTestMonitor.h index 135fe8ba5..655d06d11 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.h +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.h @@ -16,13 +16,23 @@ #include "MMDevice.h" #include "MMDeviceConstants.h" +#include #include #include #include +#include namespace mmcore { namespace internal { +enum class SeqAcqEvent { PrepareForAcq, InsertImage, AcqFinished }; + +struct SeqAcqLogEntry { + SeqAcqEvent event; + int returnCode; + std::chrono::steady_clock::time_point timestamp; +}; + class SeqAcqTestMonitor { public: explicit SeqAcqTestMonitor(const MM::Device* target) : target_(target) {} @@ -40,14 +50,9 @@ class SeqAcqTestMonitor { int OnInsertImage(); void OnAcqFinished(); - bool WaitForInsertImageCount(int n, std::chrono::milliseconds timeout); - bool WaitForAcqFinished(std::chrono::milliseconds timeout); - - bool PrepareForAcqCalled() const; - int InsertImageCount() const; - bool AcqFinishedCalled() const; - bool PrepareBeforeFirstInsert() const; - int InsertImageCountAfterError() const; + bool WaitForEvent(SeqAcqEvent event, int count, + std::chrono::milliseconds timeout); + std::vector GetLog() const; private: const MM::Device* const target_; @@ -55,16 +60,12 @@ class SeqAcqTestMonitor { mutable std::mutex mutex_; std::condition_variable cv_; - bool prepareForAcqCalled_ = false; - int insertImageCount_ = 0; - bool acqFinishedCalled_ = false; - - bool prepareBeforeFirstInsert_ = false; + std::vector log_; + int successfulInsertCount_ = 0; int injectErrorCode_ = DEVICE_OK; int injectAfterCount_ = 0; bool errorInjected_ = false; - int insertImageCountAfterError_ = 0; }; } // namespace internal From 7c8e0a5f26784c1e837d1097b0e89b447ab804d2 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Mon, 23 Mar 2026 17:59:22 -0400 Subject: [PATCH 11/17] Add test for PrepareForAcq failure handling (Assisted by Claude Code; any errors are mine.) --- MMCore/CoreCallback.cpp | 3 +- .../DeviceConformance/CameraConformance.cpp | 75 ++++++++++++++++++- .../DeviceConformance/SeqAcqTestMonitor.cpp | 13 +++- MMCore/DeviceConformance/SeqAcqTestMonitor.h | 7 +- MMCore/unittest/CameraConformance-Tests.cpp | 40 +++++++++- 5 files changed, 127 insertions(+), 11 deletions(-) diff --git a/MMCore/CoreCallback.cpp b/MMCore/CoreCallback.cpp index 31ba2ba1a..475ab86bd 100644 --- a/MMCore/CoreCallback.cpp +++ b/MMCore/CoreCallback.cpp @@ -357,8 +357,7 @@ int CoreCallback::PrepareForAcq(const MM::Device* caller) { if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { if (monitor->IsMonitoring(caller)) { - monitor->OnPrepareForAcq(); - return DEVICE_OK; + return monitor->OnPrepareForAcq(); } } diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index d71552ad7..db18371e3 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -295,7 +295,7 @@ struct CameraTestContext { result.name = slug; SeqAcqTestMonitor monitor(rawCam); - monitor.SetErrorInjection(errorCode, 3); + monitor.SetInsertImageError(errorCode, 3); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -351,6 +351,76 @@ struct CameraTestContext { return result; } + + TestResult TestPrepareErrorPropagated() { + TestResult result; + result.name = "seq-prepare-error-propagated"; + + SeqAcqTestMonitor monitor(rawCam); + monitor.SetPrepareForAcqError(DEVICE_ERR); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + bool startFailed = false; + try { + StartFinite(1, 0.0); + } catch (const CMMError&) { + startFailed = true; + } + + if (!startFailed) { + result.assertions.push_back( + {AssertionStatus::Fail, + "Camera did not propagate PrepareForAcq error", {}}); + return result; + } + + auto log = monitor.GetLog(); + if (!HasEvent(log, SeqAcqEvent::PrepareForAcq)) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed before calling PrepareForAcq", {}}); + return result; + } + + std::this_thread::sleep_for(negativeTimeout); + log = monitor.GetLog(); + + if (HasEvent(log, SeqAcqEvent::InsertImage)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "InsertImage called after PrepareForAcq error", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No InsertImage calls after PrepareForAcq error", {}}); + } + + if (HasEvent(log, SeqAcqEvent::AcqFinished)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "AcqFinished called after PrepareForAcq error", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No AcqFinished call after PrepareForAcq error", {}}); + } + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() returned true after failed start", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() returned false after failed start", {}}); + } + } + + return result; + } }; } // anonymous namespace @@ -369,6 +439,9 @@ std::vector GetCameraConformanceTests( std::vector tests; + tests.push_back({"seq-prepare-error-propagated", [ctx]() { + return ctx->TestPrepareErrorPropagated(); + }, {}}); tests.push_back({"seq-basic", [ctx]() { return ctx->TestSeqBasic(); }, {}}); diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp index 28f186304..3b2d99643 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp @@ -16,18 +16,25 @@ namespace mmcore { namespace internal { -void SeqAcqTestMonitor::SetErrorInjection(int errorCode, +void SeqAcqTestMonitor::SetPrepareForAcqError(int errorCode) { + std::lock_guard lock(mutex_); + prepareForAcqError_ = errorCode; +} + +void SeqAcqTestMonitor::SetInsertImageError(int errorCode, int afterSuccessfulCount) { std::lock_guard lock(mutex_); injectErrorCode_ = errorCode; injectAfterCount_ = afterSuccessfulCount; } -void SeqAcqTestMonitor::OnPrepareForAcq() { +int SeqAcqTestMonitor::OnPrepareForAcq() { std::lock_guard lock(mutex_); - log_.push_back({SeqAcqEvent::PrepareForAcq, DEVICE_OK, + int retCode = prepareForAcqError_; + log_.push_back({SeqAcqEvent::PrepareForAcq, retCode, std::chrono::steady_clock::now()}); cv_.notify_all(); + return retCode; } int SeqAcqTestMonitor::OnInsertImage() { diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.h b/MMCore/DeviceConformance/SeqAcqTestMonitor.h index 655d06d11..e23a67829 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.h +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.h @@ -44,9 +44,10 @@ class SeqAcqTestMonitor { return caller == target_; } - void SetErrorInjection(int errorCode, int afterSuccessfulCount); + void SetPrepareForAcqError(int errorCode); + void SetInsertImageError(int errorCode, int afterSuccessfulCount); - void OnPrepareForAcq(); + int OnPrepareForAcq(); int OnInsertImage(); void OnAcqFinished(); @@ -66,6 +67,8 @@ class SeqAcqTestMonitor { int injectErrorCode_ = DEVICE_OK; int injectAfterCount_ = 0; bool errorInjected_ = false; + + int prepareForAcqError_ = DEVICE_OK; }; } // namespace internal diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index 96cf5accb..69f47f91c 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -42,6 +42,7 @@ struct ConfigurableAsyncCamera : CCameraBase { double exposure = 10.0; bool callPrepareForAcq = true; + bool checkPrepareForAcqReturn = true; bool callAcqFinished = true; bool checkInsertImageReturn = true; bool failStartSequenceAcq = false; @@ -92,8 +93,11 @@ struct ConfigurableAsyncCamera : CCameraBase { // Thread may be left over from previous (unstopped) run if (thread_.joinable()) thread_.join(); - if (callPrepareForAcq) - GetCoreCallback()->PrepareForAcq(this); + if (callPrepareForAcq) { + int ret = GetCoreCallback()->PrepareForAcq(this); + if (checkPrepareForAcqReturn && ret != DEVICE_OK) + return ret; + } { std::lock_guard lk(mu_); running_ = true; @@ -270,7 +274,8 @@ TEST_CASE("Dependent tests are skipped when dependency warns", CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-after-count") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "skipped"); - CHECK(results["summary"]["warnings"].get() == 1); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "warning"); + CHECK(results["summary"]["warnings"].get() == 2); CHECK(results["summary"]["skipped"].get() == 6); } @@ -301,3 +306,32 @@ TEST_CASE("deviceState contains camera settings", "[CameraConformance]") { CHECK(settings["exposureSequenceable"].get() == false); CHECK(settings["multiROISupported"].get() == false); } + +TEST_CASE("Ignoring PrepareForAcq error is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.checkPrepareForAcqReturn = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests( + "cam", "seq-prepare-error-propagated")); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "fail"); +} + +TEST_CASE("Conformant camera passes PrepareForAcq error propagation test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests( + "cam", "seq-prepare-error-propagated")); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "pass"); +} From 1ab4b5fc0f021e7a752c43415798a17f9e2c7e15 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 14:01:22 -0400 Subject: [PATCH 12/17] Add testing for IsCapturing state (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 41 +++++++++++++++---- .../DeviceConformance/SeqAcqTestMonitor.cpp | 6 +-- MMCore/DeviceConformance/SeqAcqTestMonitor.h | 8 ++-- MMCore/unittest/CameraConformance-Tests.cpp | 22 +++++++++- 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index db18371e3..2c020e6be 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -142,10 +142,17 @@ int CountInsertsAfterError(const std::vector& log) { return count; } +bool WasCapturingDuringAcqFinished(const std::vector& log) { + for (const auto& e : log) + if (e.event == SeqAcqEvent::AcqFinished) + return e.isCapturing; + return false; +} + struct CameraTestContext { std::shared_ptr pCam; std::atomic& seqAcqTestMonitor; - const MM::Device* rawCam; + MM::Camera* rawCamera; std::chrono::milliseconds testTimeout; std::chrono::milliseconds negativeTimeout; @@ -185,7 +192,7 @@ struct CameraTestContext { TestResult result; result.name = "seq-basic"; - SeqAcqTestMonitor monitor(rawCam); + SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -215,7 +222,7 @@ struct CameraTestContext { TestResult result; result.name = "seq-prepare-before-insert"; - SeqAcqTestMonitor monitor(rawCam); + SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -250,7 +257,7 @@ struct CameraTestContext { TestResult result; result.name = "seq-finished-after-count"; - SeqAcqTestMonitor monitor(rawCam); + SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -268,6 +275,16 @@ struct CameraTestContext { result.assertions.push_back( {AssertionStatus::Pass, "AcqFinished called after finite sequence completed", {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } } else { AssertionResult a; a.status = AssertionStatus::Fail; @@ -294,7 +311,7 @@ struct CameraTestContext { TestResult result; result.name = slug; - SeqAcqTestMonitor monitor(rawCam); + SeqAcqTestMonitor monitor(rawCamera); monitor.SetInsertImageError(errorCode, 3); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -318,6 +335,16 @@ struct CameraTestContext { result.assertions.push_back( {AssertionStatus::Pass, std::string("AcqFinished called after ") + errorName, {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } } else { AssertionResult a; a.status = AssertionStatus::Fail; @@ -356,7 +383,7 @@ struct CameraTestContext { TestResult result; result.name = "seq-prepare-error-propagated"; - SeqAcqTestMonitor monitor(rawCam); + SeqAcqTestMonitor monitor(rawCamera); monitor.SetPrepareForAcqError(DEVICE_ERR); seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; @@ -432,7 +459,7 @@ std::vector GetCameraConformanceTests( auto ctx = std::make_shared(CameraTestContext{ pCam, seqAcqTestMonitor, - pCam->GetRawPtr(), + static_cast(pCam->GetRawPtr()), config.positiveTimeout, config.negativeTimeout}); diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp index 3b2d99643..7f0aed938 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp @@ -32,7 +32,7 @@ int SeqAcqTestMonitor::OnPrepareForAcq() { std::lock_guard lock(mutex_); int retCode = prepareForAcqError_; log_.push_back({SeqAcqEvent::PrepareForAcq, retCode, - std::chrono::steady_clock::now()}); + std::chrono::steady_clock::now(), camera_->IsCapturing()}); cv_.notify_all(); return retCode; } @@ -50,7 +50,7 @@ int SeqAcqTestMonitor::OnInsertImage() { ++successfulInsertCount_; } log_.push_back({SeqAcqEvent::InsertImage, retCode, - std::chrono::steady_clock::now()}); + std::chrono::steady_clock::now(), camera_->IsCapturing()}); cv_.notify_all(); return retCode; } @@ -58,7 +58,7 @@ int SeqAcqTestMonitor::OnInsertImage() { void SeqAcqTestMonitor::OnAcqFinished() { std::lock_guard lock(mutex_); log_.push_back({SeqAcqEvent::AcqFinished, DEVICE_OK, - std::chrono::steady_clock::now()}); + std::chrono::steady_clock::now(), camera_->IsCapturing()}); cv_.notify_all(); } diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.h b/MMCore/DeviceConformance/SeqAcqTestMonitor.h index e23a67829..88b31a2a1 100644 --- a/MMCore/DeviceConformance/SeqAcqTestMonitor.h +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.h @@ -31,17 +31,19 @@ struct SeqAcqLogEntry { SeqAcqEvent event; int returnCode; std::chrono::steady_clock::time_point timestamp; + bool isCapturing; }; class SeqAcqTestMonitor { public: - explicit SeqAcqTestMonitor(const MM::Device* target) : target_(target) {} + explicit SeqAcqTestMonitor(MM::Camera* camera) + : camera_(camera) {} SeqAcqTestMonitor(const SeqAcqTestMonitor&) = delete; SeqAcqTestMonitor& operator=(const SeqAcqTestMonitor&) = delete; bool IsMonitoring(const MM::Device* caller) const { - return caller == target_; + return caller == camera_; } void SetPrepareForAcqError(int errorCode); @@ -56,7 +58,7 @@ class SeqAcqTestMonitor { std::vector GetLog() const; private: - const MM::Device* const target_; + MM::Camera* const camera_; mutable std::mutex mutex_; std::condition_variable cv_; diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index 69f47f91c..533775445 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -44,6 +44,7 @@ struct ConfigurableAsyncCamera : CCameraBase { bool callPrepareForAcq = true; bool checkPrepareForAcqReturn = true; bool callAcqFinished = true; + bool clearCapturingBeforeAcqFinished = true; bool checkInsertImageReturn = true; bool failStartSequenceAcq = false; @@ -163,9 +164,13 @@ struct ConfigurableAsyncCamera : CCameraBase { break; } } + if (clearCapturingBeforeAcqFinished) { + std::lock_guard lk(mu_); + running_ = false; + } if (callAcqFinished) GetCoreCallback()->AcqFinished(this, 0); - { + if (!clearCapturingBeforeAcqFinished) { std::lock_guard lk(mu_); running_ = false; } @@ -229,6 +234,21 @@ TEST_CASE("Missing AcqFinished is detected by conformance test", CHECK(GetTestStatus(results, "seq-finished-after-count") == "fail"); } +TEST_CASE("IsCapturing() true during AcqFinished is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.clearCapturingBeforeAcqFinished = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-finished-after-count")); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "fail"); +} + TEST_CASE("Ignoring InsertImage error return is detected by conformance test", "[CameraConformance]") { ConfigurableAsyncCamera cam; From 77dd9ef546befb0f183ac10dee9269b4d938ce45 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 14:28:08 -0400 Subject: [PATCH 13/17] Make IsCapturing() == false a prerequisite (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 2c020e6be..06724da82 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -196,6 +196,21 @@ struct CameraTestContext { seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence acquisition", + {}}); + return result; + } + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false before starting sequence acquisition", + {}}); + } + try { StartFinite(1, 0.0); } catch (const CMMError&) { @@ -388,6 +403,21 @@ struct CameraTestContext { seqAcqTestMonitor.store(&monitor, std::memory_order_release); MonitorGuard mg{seqAcqTestMonitor, pCam}; + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence acquisition", + {}}); + return result; + } + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false before starting sequence acquisition", + {}}); + } + bool startFailed = false; try { StartFinite(1, 0.0); From 10c29f3c411a93ea8f019d8bdce8e5eb50d6124b Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 15:02:18 -0400 Subject: [PATCH 14/17] Add tests for explicit stop of sequence acq (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 141 ++++++++++++++++++ MMCore/unittest/CameraConformance-Tests.cpp | 4 +- 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 06724da82..32c3afb36 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -149,6 +149,18 @@ bool WasCapturingDuringAcqFinished(const std::vector& log) { return false; } +int CountInsertsAfterFinished(const std::vector& log) { + bool seenFinished = false; + int count = 0; + for (const auto& e : log) { + if (e.event == SeqAcqEvent::AcqFinished) + seenFinished = true; + else if (seenFinished && e.event == SeqAcqEvent::InsertImage) + ++count; + } + return count; +} + struct CameraTestContext { std::shared_ptr pCam; std::atomic& seqAcqTestMonitor; @@ -478,6 +490,129 @@ struct CameraTestContext { return result; } + + TestResult TestExplicitStop(const char* slug, bool continuous) { + TestResult result; + result.name = slug; + + SeqAcqTestMonitor monitor(rawCamera); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence " + "acquisition", + {}}); + return result; + } + } + + try { + if (continuous) + StartContinuous(0.0); + else + StartFinite(1000000, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + + if (!monitor.WaitForEvent( + SeqAcqEvent::InsertImage, 3, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera did not produce 3 images before stop", {}}); + return result; + } + + { + auto log = monitor.GetLog(); + bool allCapturing = true; + for (const auto& e : log) { + if (e.event == SeqAcqEvent::InsertImage && !e.isCapturing) { + allCapturing = false; + break; + } + } + if (allCapturing) { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was true during all InsertImage calls", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was false during an InsertImage call", + {}}); + } + } + + StopCamera(); + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true after " + "StopSequenceAcquisition", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false after " + "StopSequenceAcquisition", + {}}); + } + } + + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + "AcqFinished called after StopSequenceAcquisition", + {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "AcqFinished not called after " + "StopSequenceAcquisition", + {}}); + } + + std::this_thread::sleep_for(negativeTimeout); + auto log = monitor.GetLog(); + int afterFinished = CountInsertsAfterFinished(log); + if (afterFinished > 0) { + result.assertions.push_back( + {AssertionStatus::Fail, + std::to_string(afterFinished) + + " InsertImage call(s) after AcqFinished", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No further InsertImage calls after AcqFinished", {}}); + } + + return result; + } }; } // anonymous namespace @@ -524,6 +659,12 @@ std::vector GetCameraConformanceTests( return ctx->TestFinishedOnError("seq-finished-on-overflow-continuous", DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); }, {"seq-basic"}}); + tests.push_back({"seq-explicit-stop-finite", [ctx]() { + return ctx->TestExplicitStop("seq-explicit-stop-finite", false); + }, {"seq-basic"}}); + tests.push_back({"seq-explicit-stop-continuous", [ctx]() { + return ctx->TestExplicitStop("seq-explicit-stop-continuous", true); + }, {"seq-basic"}}); return tests; } diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index 533775445..f0bc3ecd5 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -294,9 +294,11 @@ TEST_CASE("Dependent tests are skipped when dependency warns", CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-after-count") == "skipped"); CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "skipped"); + CHECK(GetTestStatus(results, "seq-explicit-stop-finite") == "skipped"); + CHECK(GetTestStatus(results, "seq-explicit-stop-continuous") == "skipped"); CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "warning"); CHECK(results["summary"]["warnings"].get() == 2); - CHECK(results["summary"]["skipped"].get() == 6); + CHECK(results["summary"]["skipped"].get() == 8); } TEST_CASE("deviceState contains camera settings", "[CameraConformance]") { From 5a7169ce8015dd56f7f806622898c134578077da Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 15:06:48 -0400 Subject: [PATCH 15/17] Make test slugs DRY (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/CameraConformance.cpp | 26 +++++++------------ .../DeviceConformance/DeviceConformance.cpp | 1 + 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp index 32c3afb36..b655cb773 100644 --- a/MMCore/DeviceConformance/CameraConformance.cpp +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -202,7 +202,6 @@ struct CameraTestContext { TestResult TestSeqBasic() { TestResult result; - result.name = "seq-basic"; SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -247,7 +246,6 @@ struct CameraTestContext { TestResult TestPrepareBeforeInsert() { TestResult result; - result.name = "seq-prepare-before-insert"; SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -282,7 +280,6 @@ struct CameraTestContext { TestResult TestFinishedAfterCount() { TestResult result; - result.name = "seq-finished-after-count"; SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -333,10 +330,9 @@ struct CameraTestContext { return result; } - TestResult TestFinishedOnError(const char* slug, int errorCode, - const char* errorName, bool continuous) { + TestResult TestFinishedOnError(int errorCode, const char* errorName, + bool continuous) { TestResult result; - result.name = slug; SeqAcqTestMonitor monitor(rawCamera); monitor.SetInsertImageError(errorCode, 3); @@ -408,7 +404,6 @@ struct CameraTestContext { TestResult TestPrepareErrorPropagated() { TestResult result; - result.name = "seq-prepare-error-propagated"; SeqAcqTestMonitor monitor(rawCamera); monitor.SetPrepareForAcqError(DEVICE_ERR); @@ -491,9 +486,8 @@ struct CameraTestContext { return result; } - TestResult TestExplicitStop(const char* slug, bool continuous) { + TestResult TestExplicitStop(bool continuous) { TestResult result; - result.name = slug; SeqAcqTestMonitor monitor(rawCamera); seqAcqTestMonitor.store(&monitor, std::memory_order_release); @@ -644,26 +638,24 @@ std::vector GetCameraConformanceTests( return ctx->TestFinishedAfterCount(); }, {"seq-basic"}}); tests.push_back({"seq-finished-on-error-finite", [ctx]() { - return ctx->TestFinishedOnError("seq-finished-on-error-finite", - DEVICE_ERR, "DEVICE_ERR", false); + return ctx->TestFinishedOnError(DEVICE_ERR, "DEVICE_ERR", false); }, {"seq-basic"}}); tests.push_back({"seq-finished-on-error-continuous", [ctx]() { - return ctx->TestFinishedOnError("seq-finished-on-error-continuous", - DEVICE_ERR, "DEVICE_ERR", true); + return ctx->TestFinishedOnError(DEVICE_ERR, "DEVICE_ERR", true); }, {"seq-basic"}}); tests.push_back({"seq-finished-on-overflow-finite", [ctx]() { - return ctx->TestFinishedOnError("seq-finished-on-overflow-finite", + return ctx->TestFinishedOnError( DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); }, {"seq-basic"}}); tests.push_back({"seq-finished-on-overflow-continuous", [ctx]() { - return ctx->TestFinishedOnError("seq-finished-on-overflow-continuous", + return ctx->TestFinishedOnError( DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); }, {"seq-basic"}}); tests.push_back({"seq-explicit-stop-finite", [ctx]() { - return ctx->TestExplicitStop("seq-explicit-stop-finite", false); + return ctx->TestExplicitStop(false); }, {"seq-basic"}}); tests.push_back({"seq-explicit-stop-continuous", [ctx]() { - return ctx->TestExplicitStop("seq-explicit-stop-continuous", true); + return ctx->TestExplicitStop(true); }, {"seq-basic"}}); return tests; diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp index bda5080ac..2f822a15e 100644 --- a/MMCore/DeviceConformance/DeviceConformance.cpp +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -156,6 +156,7 @@ std::string RunConformanceTests( status = TestStatus::Skipped; } else { results.push_back(t.func()); + results.back().name = t.slug; status = DeriveTestStatus(results.back().assertions); } statuses.push_back(status); From 98067acde67041d9d7b696e97bbf051857aa835c Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 20:21:58 -0400 Subject: [PATCH 16/17] Add query for available tests (Assisted by Claude Code; any errors are mine.) --- .../DeviceConformance/DeviceConformance.cpp | 29 +++++++++++++ MMCore/DeviceConformance/DeviceConformance.h | 5 +++ MMCore/MMCore.cpp | 17 ++++++++ MMCore/MMCore.h | 2 + MMCore/unittest/CameraConformance-Tests.cpp | 42 +++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp index 2f822a15e..28ce80449 100644 --- a/MMCore/DeviceConformance/DeviceConformance.cpp +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -253,5 +253,34 @@ std::string RunDeviceConformanceTests( adapterName, deviceTypeStr, deviceState); } +std::string GetDeviceConformanceTestList( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config) { + const auto deviceType = device->GetType(); + const auto deviceTypeStr = ToString(deviceType); + + std::vector tests; + if (deviceType == MM::CameraDevice) { + auto pCam = std::static_pointer_cast(device); + tests = GetCameraConformanceTests(pCam, seqAcqTestMonitor, config); + } + + nlohmann::json testsJson = nlohmann::json::array(); + for (const auto& t : tests) { + nlohmann::json entry; + entry["name"] = t.slug; + entry["dependsOn"] = t.dependsOn; + testsJson.push_back(std::move(entry)); + } + + nlohmann::json j; + j["version"] = 3; + j["deviceType"] = deviceTypeStr; + j["tests"] = testsJson; + + return j.dump(2); +} + } // namespace internal } // namespace mmcore diff --git a/MMCore/DeviceConformance/DeviceConformance.h b/MMCore/DeviceConformance/DeviceConformance.h index 5e884d439..0b48fc54e 100644 --- a/MMCore/DeviceConformance/DeviceConformance.h +++ b/MMCore/DeviceConformance/DeviceConformance.h @@ -59,5 +59,10 @@ std::string RunDeviceConformanceTests( const ConformanceTestConfig& config, const char* testName); +std::string GetDeviceConformanceTestList( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config); + } // namespace internal } // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index 1e90879af..4a8bcec67 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -3089,6 +3089,23 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) return pCam->IsCapturing(); }; +/** + * Get the list of available conformance tests for a device as JSON. + * + * The returned JSON includes test names and dependency information, suitable + * for a GUI that wants to display and run tests individually. + * + * @param deviceLabel Label of the device (must be loaded and initialized). + * @return A JSON string containing the test list. + */ +std::string CMMCore::getDeviceConformanceTests(const char* deviceLabel) + MMCORE_LEGACY_THROW(CMMError) +{ + auto device = deviceManager_->GetDevice(deviceLabel); + return mmi::GetDeviceConformanceTestList(device, seqAcqTestMonitor_, + *conformanceTestConfig_); +} + /** * Run behavioral conformance tests on a device and return a JSON report. * diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index 89d290fc1..413a92f8f 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -682,6 +682,8 @@ class CMMCore /** \name Device conformance testing. */ ///@{ + std::string getDeviceConformanceTests(const char* deviceLabel) + MMCORE_LEGACY_THROW(CMMError); std::string runDeviceConformanceTests(const char* deviceLabel, const char* testName = nullptr) MMCORE_LEGACY_THROW(CMMError); ///@} diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp index f0bc3ecd5..5d7cb3543 100644 --- a/MMCore/unittest/CameraConformance-Tests.cpp +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -357,3 +357,45 @@ TEST_CASE("Conformant camera passes PrepareForAcq error propagation test", "cam", "seq-prepare-error-propagated")); CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "pass"); } + +TEST_CASE("getDeviceConformanceTests returns test list with dependencies", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto listing = nlohmann::json::parse(c.getDeviceConformanceTests("cam")); + CHECK(listing["version"].get() == 3); + CHECK(listing["deviceType"].get() == "Camera"); + REQUIRE(listing["tests"].is_array()); + REQUIRE(listing["tests"].size() > 0); + + // Every test has name and dependsOn + for (const auto& t : listing["tests"]) { + REQUIRE(t.contains("name")); + REQUIRE(t.contains("dependsOn")); + REQUIRE(t["dependsOn"].is_array()); + } + + // Check a specific dependency + bool foundPrepareBeforeInsert = false; + for (const auto& t : listing["tests"]) { + if (t["name"] == "seq-prepare-before-insert") { + foundPrepareBeforeInsert = true; + REQUIRE(t["dependsOn"].size() == 1); + CHECK(t["dependsOn"][0].get() == "seq-basic"); + } + } + CHECK(foundPrepareBeforeInsert); + + // Listing should not contain result fields + for (const auto& t : listing["tests"]) { + CHECK_FALSE(t.contains("status")); + CHECK_FALSE(t.contains("assertions")); + } + CHECK_FALSE(listing.contains("summary")); + CHECK_FALSE(listing.contains("timestamp")); +} From 97fd4675380d9880fcaf4e0361e1b5eb7aca1bff Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Tue, 24 Mar 2026 20:48:15 -0400 Subject: [PATCH 17/17] Edit API comments --- MMCore/MMCore.cpp | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index 4a8bcec67..af3de1bcd 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -3095,6 +3095,9 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) * The returned JSON includes test names and dependency information, suitable * for a GUI that wants to display and run tests individually. * + * The JSON format may change without notice (check the "version" key, which is + * incremented for incompatible changes). + * * @param deviceLabel Label of the device (must be loaded and initialized). * @return A JSON string containing the test list. */ @@ -3109,8 +3112,23 @@ std::string CMMCore::getDeviceConformanceTests(const char* deviceLabel) /** * Run behavioral conformance tests on a device and return a JSON report. * - * Currently supports camera devices, where the tests exercise the sequence - * acquisition callback protocol (PrepareForAcq, InsertImage, AcqFinished). + * Conformance tests aim to check for correct implementation of a device + * adapter, to the extent possible in fully automated tests. They do not + * attempt to test for correct hardware behavior. + * + * The focus is on catching common programming errors in device adapters, but + * not all problems can be tested. Tests may be added or refined over time, so + * a passing device adapter could fail in a future version. It is recommended + * to test device adapters against the latest MMCore. + * + * Currently, tests are available only for camera devices. + * + * Test results can depend on the current settings of the device (for example, + * exposure or ROI). This is intentional, because a device may behave correctly + * under some configurations and not under others. + * + * The JSON format may change without notice (check the "version" key, which is + * incremented for incompatible changes). * * @param deviceLabel Label of the device to test (must be loaded and * initialized). @@ -3127,13 +3145,13 @@ std::string CMMCore::runDeviceConformanceTests(const char* deviceLabel, } /** - * \brief Testing only: configure conformance tests - * + * Testing only: configure conformance tests + * * This function is designed for unit testing of MMCore itself, and its - * interface is subject to change. It is also not designed for language + * interface is subject to change. It is also not designed with language * bindings (Java, Python) in mind (at least for now). - * - * Do not use this in production code, for now. + * + * Not intended for use in production code. */ void CMMCore::setConformanceTestConfig( const mmcore::internal::ConformanceTestConfig& config) {