diff --git a/MMCore/CoreCallback.cpp b/MMCore/CoreCallback.cpp index 3b893853b..475ab86bd 100644 --- a/MMCore/CoreCallback.cpp +++ b/MMCore/CoreCallback.cpp @@ -28,6 +28,7 @@ #include "CircularBuffer.h" #include "CoreCallback.h" #include "DeviceManager.h" +#include "DeviceConformance/SeqAcqTestMonitor.h" #include "Notification.h" #include "SynchronizedConfiguration.h" @@ -227,6 +228,11 @@ int CoreCallback::InsertImage(const MM::Device* caller, const unsigned char* buf unsigned width, unsigned height, unsigned bytesPerPixel, unsigned nComponents, const char* serializedMetadata) { + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) + return monitor->OnInsertImage(); + } + Metadata origMd; if (serializedMetadata) { @@ -267,8 +273,10 @@ bool CoreCallback::InitializeImageBuffer(unsigned channels, unsigned slices, return core_->cbuf_->Initialize(w, h, pixDepth); } -int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) +int CoreCallback::AcqFinished(const MM::Device* caller, int statusCode) { + (void)statusCode; + std::shared_ptr camera; try { @@ -281,6 +289,13 @@ int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) return DEVICE_ERR; } + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) { + monitor->OnAcqFinished(); + return DEVICE_OK; + } + } + std::shared_ptr currentCamera = core_->currentCameraDevice_.lock(); @@ -340,6 +355,12 @@ int CoreCallback::AcqFinished(const MM::Device* caller, int /*statusCode*/) int CoreCallback::PrepareForAcq(const MM::Device* caller) { + if (auto* monitor = core_->seqAcqTestMonitor_.load(std::memory_order_acquire)) { + if (monitor->IsMonitoring(caller)) { + return monitor->OnPrepareForAcq(); + } + } + if (core_->autoShutter_) { std::shared_ptr shutter = diff --git a/MMCore/DeviceConformance/CameraConformance.cpp b/MMCore/DeviceConformance/CameraConformance.cpp new file mode 100644 index 000000000..b655cb773 --- /dev/null +++ b/MMCore/DeviceConformance/CameraConformance.cpp @@ -0,0 +1,665 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "CameraConformance.h" + +#include "DeviceManager.h" +#include "Devices/CameraInstance.h" +#include "Error.h" + +#include "MMDeviceConstants.h" + +#include + +#include +#include + +namespace mmcore { +namespace internal { + +nlohmann::json CollectCameraState( + std::shared_ptr camera) { + nlohmann::json s = nlohmann::json::object(); + DeviceModuleLockGuard guard(camera); + + try { s["exposure"] = camera->GetExposure(); } + catch (...) { s["exposure"] = nullptr; } + + try { s["binning"] = camera->GetBinning(); } + catch (...) { s["binning"] = nullptr; } + + try { + unsigned x, y, w, h; + camera->GetROI(x, y, w, h); + s["roi"] = {{"x", x}, {"y", y}, {"width", w}, {"height", h}}; + } catch (...) { s["roi"] = nullptr; } + + try { s["imageWidth"] = camera->GetImageWidth(); } + catch (...) { s["imageWidth"] = nullptr; } + + try { s["imageHeight"] = camera->GetImageHeight(); } + catch (...) { s["imageHeight"] = nullptr; } + + try { s["imageBytesPerPixel"] = camera->GetImageBytesPerPixel(); } + catch (...) { s["imageBytesPerPixel"] = nullptr; } + + try { s["bitDepth"] = camera->GetBitDepth(); } + catch (...) { s["bitDepth"] = nullptr; } + + try { s["numberOfComponents"] = camera->GetNumberOfComponents(); } + catch (...) { s["numberOfComponents"] = nullptr; } + + try { + unsigned nCh = camera->GetNumberOfChannels(); + nlohmann::json channels = nlohmann::json::array(); + for (unsigned i = 0; i < nCh; ++i) + channels.push_back(camera->GetChannelName(i)); + s["channels"] = channels; + } catch (...) { s["channels"] = nullptr; } + + try { s["imageBufferSize"] = camera->GetImageBufferSize(); } + catch (...) { s["imageBufferSize"] = nullptr; } + + try { s["multiROISupported"] = camera->SupportsMultiROI(); } + catch (...) { s["multiROISupported"] = nullptr; } + + try { s["multiROIEnabled"] = camera->IsMultiROISet(); } + catch (...) { s["multiROIEnabled"] = nullptr; } + + try { + if (camera->IsMultiROISet()) { + unsigned count = 0; + camera->GetMultiROI(nullptr, nullptr, nullptr, nullptr, &count); + std::vector xs(count), ys(count), ws(count), hs(count); + camera->GetMultiROI(xs.data(), ys.data(), ws.data(), hs.data(), + &count); + nlohmann::json rois = nlohmann::json::array(); + for (unsigned i = 0; i < count; ++i) + rois.push_back( + {{"x", xs[i]}, {"y", ys[i]}, + {"width", ws[i]}, {"height", hs[i]}}); + s["multiROIs"] = rois; + } + } catch (...) { s["multiROIs"] = nullptr; } + + bool isSeq = false; + try { + camera->IsExposureSequenceable(isSeq); + s["exposureSequenceable"] = isSeq; + } catch (...) { s["exposureSequenceable"] = nullptr; } + + if (isSeq) { + try { + long maxLen = 0; + camera->GetExposureSequenceMaxLength(maxLen); + s["exposureSequenceMaxLength"] = maxLen; + } catch (...) { s["exposureSequenceMaxLength"] = nullptr; } + } + + return s; +} + +namespace { + +bool HasEvent(const std::vector& log, SeqAcqEvent event) { + for (const auto& e : log) + if (e.event == event) + return true; + return false; +} + +bool PrepareBeforeFirstInsert(const std::vector& log) { + for (const auto& e : log) { + if (e.event == SeqAcqEvent::PrepareForAcq) + return true; + if (e.event == SeqAcqEvent::InsertImage) + return false; + } + return false; +} + +int CountInsertsAfterError(const std::vector& log) { + bool seenError = false; + int count = 0; + for (const auto& e : log) { + if (e.event != SeqAcqEvent::InsertImage) + continue; + if (seenError) + ++count; + else if (e.returnCode != DEVICE_OK) + seenError = true; + } + return count; +} + +bool WasCapturingDuringAcqFinished(const std::vector& log) { + for (const auto& e : log) + if (e.event == SeqAcqEvent::AcqFinished) + return e.isCapturing; + return false; +} + +int CountInsertsAfterFinished(const std::vector& log) { + bool seenFinished = false; + int count = 0; + for (const auto& e : log) { + if (e.event == SeqAcqEvent::AcqFinished) + seenFinished = true; + else if (seenFinished && e.event == SeqAcqEvent::InsertImage) + ++count; + } + return count; +} + +struct CameraTestContext { + std::shared_ptr pCam; + std::atomic& seqAcqTestMonitor; + MM::Camera* rawCamera; + std::chrono::milliseconds testTimeout; + std::chrono::milliseconds negativeTimeout; + + struct MonitorGuard { + std::atomic& atom; + std::shared_ptr cam; + ~MonitorGuard() { + try { + DeviceModuleLockGuard g(cam); + cam->StopSequenceAcquisition(); + } catch (...) {} + atom.store(nullptr, std::memory_order_release); + } + }; + + void StartFinite(long numImages, double intervalMs) { + DeviceModuleLockGuard guard(pCam); + int nRet = pCam->StartSequenceAcquisition(numImages, intervalMs, false); + if (nRet != DEVICE_OK) + throw CMMError("Camera failed to start finite sequence acquisition"); + } + + void StartContinuous(double intervalMs) { + DeviceModuleLockGuard guard(pCam); + int nRet = pCam->StartSequenceAcquisition(intervalMs); + if (nRet != DEVICE_OK) + throw CMMError( + "Camera failed to start continuous sequence acquisition"); + } + + void StopCamera() { + DeviceModuleLockGuard guard(pCam); + pCam->StopSequenceAcquisition(); + } + + TestResult TestSeqBasic() { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence acquisition", + {}}); + return result; + } + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false before starting sequence acquisition", + {}}); + } + + try { + StartFinite(1, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + + if (monitor.WaitForEvent(SeqAcqEvent::InsertImage, 1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + "Camera produced 1 image via sequence acquisition", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "No image arrived from sequence acquisition", {}}); + } + + return result; + } + + TestResult TestPrepareBeforeInsert() { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + try { + StartFinite(5, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 5, testTimeout); + + auto log = monitor.GetLog(); + if (!HasEvent(log, SeqAcqEvent::PrepareForAcq)) { + result.assertions.push_back( + {AssertionStatus::Fail, "PrepareForAcq was not called", {}}); + } else if (!PrepareBeforeFirstInsert(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "PrepareForAcq was called after InsertImage", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "PrepareForAcq called before first InsertImage", {}}); + } + + return result; + } + + TestResult TestFinishedAfterCount() { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + try { + StartFinite(5, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 5, testTimeout); + + if (monitor.WaitForEvent(SeqAcqEvent::AcqFinished, 1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + "AcqFinished called after finite sequence completed", {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } + } else { + AssertionResult a; + a.status = AssertionStatus::Fail; + a.message = + "AcqFinished not called after finite sequence (5 frames)"; + StopCamera(); + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { + a.details.push_back( + "AcqFinished was called after stopSequenceAcquisition"); + } else { + a.details.push_back( + "AcqFinished was not called even after " + "stopSequenceAcquisition"); + } + result.assertions.push_back(std::move(a)); + } + + return result; + } + + TestResult TestFinishedOnError(int errorCode, const char* errorName, + bool continuous) { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + monitor.SetInsertImageError(errorCode, 3); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + try { + if (continuous) + StartContinuous(0.0); + else + StartFinite(1000000, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + + monitor.WaitForEvent(SeqAcqEvent::InsertImage, 4, testTimeout); + + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + std::string("AcqFinished called after ") + errorName, {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } + } else { + AssertionResult a; + a.status = AssertionStatus::Fail; + a.message = + std::string("AcqFinished not called after ") + errorName; + StopCamera(); + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { + a.details.push_back( + "AcqFinished was called after stopSequenceAcquisition"); + } else { + a.details.push_back( + "AcqFinished was not called even after " + "stopSequenceAcquisition"); + } + result.assertions.push_back(std::move(a)); + } + + std::this_thread::sleep_for(negativeTimeout); + auto log = monitor.GetLog(); + int afterError = CountInsertsAfterError(log); + if (afterError > 0) { + result.assertions.push_back( + {AssertionStatus::Fail, std::to_string(afterError) + + " InsertImage call(s) after error return", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No further InsertImage calls after error", {}}); + } + + return result; + } + + TestResult TestPrepareErrorPropagated() { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + monitor.SetPrepareForAcqError(DEVICE_ERR); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence acquisition", + {}}); + return result; + } + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false before starting sequence acquisition", + {}}); + } + + bool startFailed = false; + try { + StartFinite(1, 0.0); + } catch (const CMMError&) { + startFailed = true; + } + + if (!startFailed) { + result.assertions.push_back( + {AssertionStatus::Fail, + "Camera did not propagate PrepareForAcq error", {}}); + return result; + } + + auto log = monitor.GetLog(); + if (!HasEvent(log, SeqAcqEvent::PrepareForAcq)) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed before calling PrepareForAcq", {}}); + return result; + } + + std::this_thread::sleep_for(negativeTimeout); + log = monitor.GetLog(); + + if (HasEvent(log, SeqAcqEvent::InsertImage)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "InsertImage called after PrepareForAcq error", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No InsertImage calls after PrepareForAcq error", {}}); + } + + if (HasEvent(log, SeqAcqEvent::AcqFinished)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "AcqFinished called after PrepareForAcq error", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No AcqFinished call after PrepareForAcq error", {}}); + } + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() returned true after failed start", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() returned false after failed start", {}}); + } + } + + return result; + } + + TestResult TestExplicitStop(bool continuous) { + TestResult result; + + SeqAcqTestMonitor monitor(rawCamera); + seqAcqTestMonitor.store(&monitor, std::memory_order_release); + MonitorGuard mg{seqAcqTestMonitor, pCam}; + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Warning, + "IsCapturing() was true before starting sequence " + "acquisition", + {}}); + return result; + } + } + + try { + if (continuous) + StartContinuous(0.0); + else + StartFinite(1000000, 0.0); + } catch (const CMMError&) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera failed to start sequence acquisition", {}}); + return result; + } + + if (!monitor.WaitForEvent( + SeqAcqEvent::InsertImage, 3, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Warning, + "Camera did not produce 3 images before stop", {}}); + return result; + } + + { + auto log = monitor.GetLog(); + bool allCapturing = true; + for (const auto& e : log) { + if (e.event == SeqAcqEvent::InsertImage && !e.isCapturing) { + allCapturing = false; + break; + } + } + if (allCapturing) { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was true during all InsertImage calls", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was false during an InsertImage call", + {}}); + } + } + + StopCamera(); + + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true after " + "StopSequenceAcquisition", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false after " + "StopSequenceAcquisition", + {}}); + } + } + + if (monitor.WaitForEvent( + SeqAcqEvent::AcqFinished, 1, testTimeout)) { + result.assertions.push_back( + {AssertionStatus::Pass, + "AcqFinished called after StopSequenceAcquisition", + {}}); + auto log = monitor.GetLog(); + if (WasCapturingDuringAcqFinished(log)) { + result.assertions.push_back( + {AssertionStatus::Fail, + "IsCapturing() was true during AcqFinished", {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "IsCapturing() was false during AcqFinished", {}}); + } + } else { + result.assertions.push_back( + {AssertionStatus::Fail, + "AcqFinished not called after " + "StopSequenceAcquisition", + {}}); + } + + std::this_thread::sleep_for(negativeTimeout); + auto log = monitor.GetLog(); + int afterFinished = CountInsertsAfterFinished(log); + if (afterFinished > 0) { + result.assertions.push_back( + {AssertionStatus::Fail, + std::to_string(afterFinished) + + " InsertImage call(s) after AcqFinished", + {}}); + } else { + result.assertions.push_back( + {AssertionStatus::Pass, + "No further InsertImage calls after AcqFinished", {}}); + } + + return result; + } +}; + +} // anonymous namespace + +std::vector GetCameraConformanceTests( + std::shared_ptr pCam, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config) { + auto ctx = std::make_shared(CameraTestContext{ + pCam, + seqAcqTestMonitor, + static_cast(pCam->GetRawPtr()), + config.positiveTimeout, + config.negativeTimeout}); + + + std::vector tests; + + tests.push_back({"seq-prepare-error-propagated", [ctx]() { + return ctx->TestPrepareErrorPropagated(); + }, {}}); + tests.push_back({"seq-basic", [ctx]() { + return ctx->TestSeqBasic(); + }, {}}); + tests.push_back({"seq-prepare-before-insert", [ctx]() { + return ctx->TestPrepareBeforeInsert(); + }, {"seq-basic"}}); + tests.push_back({"seq-finished-after-count", [ctx]() { + return ctx->TestFinishedAfterCount(); + }, {"seq-basic"}}); + tests.push_back({"seq-finished-on-error-finite", [ctx]() { + return ctx->TestFinishedOnError(DEVICE_ERR, "DEVICE_ERR", false); + }, {"seq-basic"}}); + tests.push_back({"seq-finished-on-error-continuous", [ctx]() { + return ctx->TestFinishedOnError(DEVICE_ERR, "DEVICE_ERR", true); + }, {"seq-basic"}}); + tests.push_back({"seq-finished-on-overflow-finite", [ctx]() { + return ctx->TestFinishedOnError( + DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", false); + }, {"seq-basic"}}); + tests.push_back({"seq-finished-on-overflow-continuous", [ctx]() { + return ctx->TestFinishedOnError( + DEVICE_BUFFER_OVERFLOW, "DEVICE_BUFFER_OVERFLOW", true); + }, {"seq-basic"}}); + tests.push_back({"seq-explicit-stop-finite", [ctx]() { + return ctx->TestExplicitStop(false); + }, {"seq-basic"}}); + tests.push_back({"seq-explicit-stop-continuous", [ctx]() { + return ctx->TestExplicitStop(true); + }, {"seq-basic"}}); + + return tests; +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/CameraConformance.h b/MMCore/DeviceConformance/CameraConformance.h new file mode 100644 index 000000000..911e677ed --- /dev/null +++ b/MMCore/DeviceConformance/CameraConformance.h @@ -0,0 +1,38 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "DeviceConformance.h" + +#include +#include +#include + +#include + +namespace mmcore { +namespace internal { + +class CameraInstance; + +nlohmann::json CollectCameraState( + std::shared_ptr camera); + +std::vector GetCameraConformanceTests( + std::shared_ptr camera, + std::atomic& testMonitor, + const ConformanceTestConfig& config); + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/ConformanceTestConfig.h b/MMCore/DeviceConformance/ConformanceTestConfig.h new file mode 100644 index 000000000..ca5d1696b --- /dev/null +++ b/MMCore/DeviceConformance/ConformanceTestConfig.h @@ -0,0 +1,27 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include + +namespace mmcore { +namespace internal { + +struct ConformanceTestConfig { + std::chrono::milliseconds positiveTimeout{10000}; + std::chrono::milliseconds negativeTimeout{2000}; +}; + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/DeviceConformance.cpp b/MMCore/DeviceConformance/DeviceConformance.cpp new file mode 100644 index 000000000..28ce80449 --- /dev/null +++ b/MMCore/DeviceConformance/DeviceConformance.cpp @@ -0,0 +1,286 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "DeviceConformance.h" + +#include "CameraConformance.h" +#include "CoreUtils.h" +#include "DeviceManager.h" +#include "Devices/CameraInstance.h" +#include "Devices/DeviceInstance.h" +#include "Error.h" + +#include "MMDeviceConstants.h" + +#include + +#include +#include +#include +#include +#include + +namespace mmcore { +namespace internal { + +namespace { + +std::string FormatISO8601(std::chrono::system_clock::time_point tp) { + auto time = std::chrono::system_clock::to_time_t(tp); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &time); +#else + gmtime_r(&time, &tm); +#endif + std::ostringstream ss; + ss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); + return ss.str(); +} + +const char* AssertionStatusToString(AssertionStatus s) { + switch (s) { + case AssertionStatus::Pass: return "pass"; + case AssertionStatus::Fail: return "fail"; + case AssertionStatus::Warning: return "warning"; + } + return "unknown"; +} + +const char* TestStatusToString(TestStatus s) { + switch (s) { + case TestStatus::Pass: return "pass"; + case TestStatus::Fail: return "fail"; + case TestStatus::Warning: return "warning"; + case TestStatus::Skipped: return "skipped"; + } + return "unknown"; +} + +TestStatus DeriveTestStatus(const std::vector& assertions) { + bool anyWarning = false; + for (const auto& a : assertions) { + if (a.status == AssertionStatus::Fail) + return TestStatus::Fail; + if (a.status == AssertionStatus::Warning) + anyWarning = true; + } + return anyWarning ? TestStatus::Warning : TestStatus::Pass; +} + +nlohmann::json AssertionToJson(const AssertionResult& a) { + nlohmann::json j; + j["status"] = AssertionStatusToString(a.status); + j["message"] = a.message; + if (!a.details.empty()) + j["details"] = a.details; + return j; +} + +nlohmann::json TestToJson(const TestResult& t, TestStatus status) { + nlohmann::json j; + j["name"] = t.name; + j["status"] = TestStatusToString(status); + nlohmann::json assertions = nlohmann::json::array(); + for (const auto& a : t.assertions) + assertions.push_back(AssertionToJson(a)); + j["assertions"] = assertions; + return j; +} + +std::string RunConformanceTests( + const std::vector& tests, + const char* testName, + const std::string& deviceLabel, + const std::string& deviceName, + const std::string& adapterName, + const std::string& deviceType, + const nlohmann::json& deviceState) { + using namespace std::chrono; + + std::string selectedTest; + if (testName && testName[0] != '\0') + selectedTest = testName; + + if (!selectedTest.empty()) { + bool found = false; + for (const auto& t : tests) { + if (selectedTest == t.slug) { + found = true; + break; + } + } + if (!found) { + throw CMMError("Unknown conformance test: " + selectedTest); + } + } + + const auto startTime = system_clock::now(); + const auto startSteady = steady_clock::now(); + + std::vector results; + std::vector statuses; + std::unordered_map completedStatuses; + + for (const auto& t : tests) { + if (!selectedTest.empty() && selectedTest != t.slug) + continue; + + bool skip = false; + if (selectedTest.empty()) { + for (const auto& dep : t.dependsOn) { + auto it = completedStatuses.find(dep); + if (it == completedStatuses.end() || + it->second != TestStatus::Pass) { + skip = true; + break; + } + } + } + + TestStatus status; + if (skip) { + TestResult r; + r.name = t.slug; + results.push_back(std::move(r)); + status = TestStatus::Skipped; + } else { + results.push_back(t.func()); + results.back().name = t.slug; + status = DeriveTestStatus(results.back().assertions); + } + statuses.push_back(status); + completedStatuses[t.slug] = status; + } + + const auto endSteady = steady_clock::now(); + double durationMs = + duration_cast>(endSteady - startSteady) + .count(); + + int passedCount = 0, failedCount = 0, warningCount = 0, skippedCount = 0; + for (auto s : statuses) { + switch (s) { + case TestStatus::Pass: ++passedCount; break; + case TestStatus::Fail: ++failedCount; break; + case TestStatus::Warning: ++warningCount; break; + case TestStatus::Skipped: ++skippedCount; break; + } + } + + nlohmann::json testsJson = nlohmann::json::array(); + for (size_t i = 0; i < results.size(); ++i) + testsJson.push_back(TestToJson(results[i], statuses[i])); + + nlohmann::json j; + j["version"] = 3; + j["timestamp"] = FormatISO8601(startTime); + j["device"] = { + {"label", deviceLabel}, + {"name", deviceName}, + {"library", adapterName}, + }; + j["deviceType"] = deviceType; + j["deviceState"] = deviceState; + j["tests"] = testsJson; + j["summary"] = { + {"total", static_cast(results.size())}, + {"passed", passedCount}, + {"failed", failedCount}, + {"warnings", warningCount}, + {"skipped", skippedCount}, + {"durationMs", durationMs}, + }; + + return j.dump(2); +} + +} // anonymous namespace + +nlohmann::json CollectDeviceProperties( + std::shared_ptr device) { + nlohmann::json props = nlohmann::json::object(); + DeviceModuleLockGuard guard(device); + for (const auto& name : device->GetPropertyNames()) { + try { + props[name] = device->GetProperty(name); + } catch (...) { + props[name] = nullptr; + } + } + return props; +} + +std::string RunDeviceConformanceTests( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config, + const char* testName) { + const auto deviceLabel = device->GetLabel(); + const auto deviceName = device->GetName(); + const auto adapterName = device->GetAdapterModule()->GetName(); + const auto deviceType = device->GetType(); + const auto deviceTypeStr = ToString(deviceType); + + nlohmann::json deviceState; + deviceState["properties"] = CollectDeviceProperties(device); + + std::vector tests; + if (deviceType == MM::CameraDevice) { + auto pCam = std::static_pointer_cast(device); + { + DeviceModuleLockGuard guard(pCam); + if (pCam->IsCapturing()) + throw CMMError( + "Not allowed during sequence acquisition", + MMERR_NotAllowedDuringSequenceAcquisition); + } + deviceState["settings"] = CollectCameraState(pCam); + tests = GetCameraConformanceTests(pCam, seqAcqTestMonitor, config); + } + + return RunConformanceTests(tests, testName, deviceLabel, deviceName, + adapterName, deviceTypeStr, deviceState); +} + +std::string GetDeviceConformanceTestList( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config) { + const auto deviceType = device->GetType(); + const auto deviceTypeStr = ToString(deviceType); + + std::vector tests; + if (deviceType == MM::CameraDevice) { + auto pCam = std::static_pointer_cast(device); + tests = GetCameraConformanceTests(pCam, seqAcqTestMonitor, config); + } + + nlohmann::json testsJson = nlohmann::json::array(); + for (const auto& t : tests) { + nlohmann::json entry; + entry["name"] = t.slug; + entry["dependsOn"] = t.dependsOn; + testsJson.push_back(std::move(entry)); + } + + nlohmann::json j; + j["version"] = 3; + j["deviceType"] = deviceTypeStr; + j["tests"] = testsJson; + + return j.dump(2); +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/DeviceConformance.h b/MMCore/DeviceConformance/DeviceConformance.h new file mode 100644 index 000000000..0b48fc54e --- /dev/null +++ b/MMCore/DeviceConformance/DeviceConformance.h @@ -0,0 +1,68 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "ConformanceTestConfig.h" +#include "SeqAcqTestMonitor.h" + +#include +#include +#include +#include +#include + +#include + +namespace mmcore { +namespace internal { + +class DeviceInstance; + +enum class AssertionStatus { Pass, Fail, Warning }; + +struct AssertionResult { + AssertionStatus status; + std::string message; + std::vector details; +}; + +enum class TestStatus { Pass, Fail, Warning, Skipped }; + +struct TestResult { + std::string name; + std::vector assertions; +}; + +struct TestEntry { + std::string slug; + std::function func; + std::vector dependsOn; +}; + +nlohmann::json CollectDeviceProperties( + std::shared_ptr device); + +std::string RunDeviceConformanceTests( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config, + const char* testName); + +std::string GetDeviceConformanceTestList( + std::shared_ptr device, + std::atomic& seqAcqTestMonitor, + const ConformanceTestConfig& config); + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp new file mode 100644 index 000000000..7f0aed938 --- /dev/null +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.cpp @@ -0,0 +1,83 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#include "SeqAcqTestMonitor.h" + +namespace mmcore { +namespace internal { + +void SeqAcqTestMonitor::SetPrepareForAcqError(int errorCode) { + std::lock_guard lock(mutex_); + prepareForAcqError_ = errorCode; +} + +void SeqAcqTestMonitor::SetInsertImageError(int errorCode, + int afterSuccessfulCount) { + std::lock_guard lock(mutex_); + injectErrorCode_ = errorCode; + injectAfterCount_ = afterSuccessfulCount; +} + +int SeqAcqTestMonitor::OnPrepareForAcq() { + std::lock_guard lock(mutex_); + int retCode = prepareForAcqError_; + log_.push_back({SeqAcqEvent::PrepareForAcq, retCode, + std::chrono::steady_clock::now(), camera_->IsCapturing()}); + cv_.notify_all(); + return retCode; +} + +int SeqAcqTestMonitor::OnInsertImage() { + std::lock_guard lock(mutex_); + int retCode = DEVICE_OK; + if (injectErrorCode_ != DEVICE_OK && + successfulInsertCount_ >= injectAfterCount_) { + errorInjected_ = true; + } + if (errorInjected_) { + retCode = injectErrorCode_; + } else { + ++successfulInsertCount_; + } + log_.push_back({SeqAcqEvent::InsertImage, retCode, + std::chrono::steady_clock::now(), camera_->IsCapturing()}); + cv_.notify_all(); + return retCode; +} + +void SeqAcqTestMonitor::OnAcqFinished() { + std::lock_guard lock(mutex_); + log_.push_back({SeqAcqEvent::AcqFinished, DEVICE_OK, + std::chrono::steady_clock::now(), camera_->IsCapturing()}); + cv_.notify_all(); +} + +bool SeqAcqTestMonitor::WaitForEvent(SeqAcqEvent event, int count, + std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { + int n = 0; + for (const auto& entry : log_) + if (entry.event == event) + ++n; + return n >= count; + }); +} + +std::vector SeqAcqTestMonitor::GetLog() const { + std::lock_guard lock(mutex_); + return log_; +} + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/DeviceConformance/SeqAcqTestMonitor.h b/MMCore/DeviceConformance/SeqAcqTestMonitor.h new file mode 100644 index 000000000..88b31a2a1 --- /dev/null +++ b/MMCore/DeviceConformance/SeqAcqTestMonitor.h @@ -0,0 +1,77 @@ +// COPYRIGHT: 2026, Board of Regents of the University of Wisconsin System +// +// LICENSE: This file is distributed under the "Lesser GPL" (LGPL) license. +// License text is included with the source distribution. +// +// This file is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +// IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES. + +#pragma once + +#include "MMDevice.h" +#include "MMDeviceConstants.h" + +#include +#include +#include +#include +#include + +namespace mmcore { +namespace internal { + +enum class SeqAcqEvent { PrepareForAcq, InsertImage, AcqFinished }; + +struct SeqAcqLogEntry { + SeqAcqEvent event; + int returnCode; + std::chrono::steady_clock::time_point timestamp; + bool isCapturing; +}; + +class SeqAcqTestMonitor { +public: + explicit SeqAcqTestMonitor(MM::Camera* camera) + : camera_(camera) {} + + SeqAcqTestMonitor(const SeqAcqTestMonitor&) = delete; + SeqAcqTestMonitor& operator=(const SeqAcqTestMonitor&) = delete; + + bool IsMonitoring(const MM::Device* caller) const { + return caller == camera_; + } + + void SetPrepareForAcqError(int errorCode); + void SetInsertImageError(int errorCode, int afterSuccessfulCount); + + int OnPrepareForAcq(); + int OnInsertImage(); + void OnAcqFinished(); + + bool WaitForEvent(SeqAcqEvent event, int count, + std::chrono::milliseconds timeout); + std::vector GetLog() const; + +private: + MM::Camera* const camera_; + + mutable std::mutex mutex_; + std::condition_variable cv_; + + std::vector log_; + int successfulInsertCount_ = 0; + + int injectErrorCode_ = DEVICE_OK; + int injectAfterCount_ = 0; + bool errorInjected_ = false; + + int prepareForAcqError_ = DEVICE_OK; +}; + +} // namespace internal +} // namespace mmcore diff --git a/MMCore/MMCore.cpp b/MMCore/MMCore.cpp index a1e33dbb1..af3de1bcd 100644 --- a/MMCore/MMCore.cpp +++ b/MMCore/MMCore.cpp @@ -51,6 +51,7 @@ #include "MMEventCallback.h" #include "NotificationQueue.h" #include "PluginManager.h" +#include "DeviceConformance/DeviceConformance.h" #include "SynchronizedConfiguration.h" #include "DeviceUtils.h" @@ -134,6 +135,7 @@ CMMCore::CMMCore() : cbuf_(std::make_unique( (sizeof(void*) > 4) ? 250u : 25u)), callback_(std::make_unique(this)), + conformanceTestConfig_(std::make_unique()), pluginManager_(std::make_shared()), deviceManager_(std::make_shared()), stateCache_(std::make_unique()) @@ -3087,6 +3089,75 @@ bool CMMCore::isSequenceRunning(const char* label) MMCORE_LEGACY_THROW(CMMError) return pCam->IsCapturing(); }; +/** + * Get the list of available conformance tests for a device as JSON. + * + * The returned JSON includes test names and dependency information, suitable + * for a GUI that wants to display and run tests individually. + * + * The JSON format may change without notice (check the "version" key, which is + * incremented for incompatible changes). + * + * @param deviceLabel Label of the device (must be loaded and initialized). + * @return A JSON string containing the test list. + */ +std::string CMMCore::getDeviceConformanceTests(const char* deviceLabel) + MMCORE_LEGACY_THROW(CMMError) +{ + auto device = deviceManager_->GetDevice(deviceLabel); + return mmi::GetDeviceConformanceTestList(device, seqAcqTestMonitor_, + *conformanceTestConfig_); +} + +/** + * Run behavioral conformance tests on a device and return a JSON report. + * + * Conformance tests aim to check for correct implementation of a device + * adapter, to the extent possible in fully automated tests. They do not + * attempt to test for correct hardware behavior. + * + * The focus is on catching common programming errors in device adapters, but + * not all problems can be tested. Tests may be added or refined over time, so + * a passing device adapter could fail in a future version. It is recommended + * to test device adapters against the latest MMCore. + * + * Currently, tests are available only for camera devices. + * + * Test results can depend on the current settings of the device (for example, + * exposure or ROI). This is intentional, because a device may behave correctly + * under some configurations and not under others. + * + * The JSON format may change without notice (check the "version" key, which is + * incremented for incompatible changes). + * + * @param deviceLabel Label of the device to test (must be loaded and + * initialized). + * @param testName Slug name of a single test to run, or null/empty to run + * all tests. + * @return A JSON string containing the test results. + */ +std::string CMMCore::runDeviceConformanceTests(const char* deviceLabel, + const char* testName) MMCORE_LEGACY_THROW(CMMError) +{ + auto device = deviceManager_->GetDevice(deviceLabel); + return mmi::RunDeviceConformanceTests(device, seqAcqTestMonitor_, + *conformanceTestConfig_, testName); +} + +/** + * Testing only: configure conformance tests + * + * This function is designed for unit testing of MMCore itself, and its + * interface is subject to change. It is also not designed with language + * bindings (Java, Python) in mind (at least for now). + * + * Not intended for use in production code. + */ +void CMMCore::setConformanceTestConfig( + const mmcore::internal::ConformanceTestConfig& config) { + *conformanceTestConfig_ = config; +} + /** * Gets the last image from the circular buffer. * Returns 0 if the buffer is empty. diff --git a/MMCore/MMCore.h b/MMCore/MMCore.h index 2da918627..413a92f8f 100644 --- a/MMCore/MMCore.h +++ b/MMCore/MMCore.h @@ -56,6 +56,7 @@ #include "MMDevice.h" #include "MMDeviceConstants.h" +#include #include #include #include @@ -95,6 +96,8 @@ namespace internal { class DeviceManager; class LogManager; class NotificationQueue; + struct ConformanceTestConfig; + class SeqAcqTestMonitor; } // namespace internal } // namespace mmcore @@ -677,11 +680,21 @@ class CMMCore std::vector getLoadedPeripheralDevices(const char* hubLabel) MMCORE_LEGACY_THROW(CMMError); ///@} + /** \name Device conformance testing. */ + ///@{ + std::string getDeviceConformanceTests(const char* deviceLabel) + MMCORE_LEGACY_THROW(CMMError); + std::string runDeviceConformanceTests(const char* deviceLabel, + const char* testName = nullptr) MMCORE_LEGACY_THROW(CMMError); + ///@} + #if !defined(SWIGJAVA) && !defined(SWIGPYTHON) /** \name Testing */ ///@{ void loadMockDeviceAdapter(const char* name, MockDeviceAdapter* implementation) MMCORE_LEGACY_THROW(CMMError); + void setConformanceTestConfig( + const mmcore::internal::ConformanceTestConfig& config); ///@} #endif @@ -720,6 +733,9 @@ class CMMCore std::unique_ptr cbuf_; std::unique_ptr callback_; + std::atomic seqAcqTestMonitor_{nullptr}; + std::unique_ptr conformanceTestConfig_; + std::shared_ptr pluginManager_; std::shared_ptr deviceManager_; std::map errorText_; diff --git a/MMCore/meson.build b/MMCore/meson.build index 0ccb60e1a..c472887e8 100644 --- a/MMCore/meson.build +++ b/MMCore/meson.build @@ -27,6 +27,8 @@ mmdevice_proj = subproject( ) mmdevice_dep = mmdevice_proj.get_variable('mmdevice_dep') +nlohmann_json_dep = dependency('nlohmann_json') + mmcore_sources = files( 'CircularBuffer.cpp', 'Configuration.cpp', @@ -62,6 +64,9 @@ mmcore_sources = files( 'LogManager.cpp', 'MMCore.cpp', 'PluginManager.cpp', + 'DeviceConformance/CameraConformance.cpp', + 'DeviceConformance/DeviceConformance.cpp', + 'DeviceConformance/SeqAcqTestMonitor.cpp', 'Semaphore.cpp', 'Task.cpp', 'TaskSet.cpp', @@ -96,6 +101,7 @@ mmcore_lib = static_library( include_directories: mmcore_include_dir, dependencies: [ mmdevice_dep, + nlohmann_json_dep, dependency('threads'), dependency('dl', required: false), ], diff --git a/MMCore/subprojects/.gitignore b/MMCore/subprojects/.gitignore index d12f32e5d..2dae20e18 100644 --- a/MMCore/subprojects/.gitignore +++ b/MMCore/subprojects/.gitignore @@ -13,3 +13,4 @@ # Do not ignore wraps we provide !/catch2.wrap !/mmdevice.wrap +!/nlohmann_json.wrap diff --git a/MMCore/subprojects/nlohmann_json.wrap b/MMCore/subprojects/nlohmann_json.wrap new file mode 100644 index 000000000..f9bb22de1 --- /dev/null +++ b/MMCore/subprojects/nlohmann_json.wrap @@ -0,0 +1,11 @@ +[wrap-file] +directory = nlohmann_json-3.12.0 +lead_directory_missing = true +source_url = https://github.com/nlohmann/json/releases/download/v3.12.0/include.zip +source_filename = nlohmann_json-3.12.0.zip +source_hash = b8cb0ef2dd7f57f18933997c9934bb1fa962594f701cd5a8d3c2c80541559372 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/nlohmann_json_3.12.0-1/nlohmann_json-3.12.0.zip +wrapdb_version = 3.12.0-1 + +[provide] +nlohmann_json = nlohmann_json_dep diff --git a/MMCore/unittest/CameraConformance-Tests.cpp b/MMCore/unittest/CameraConformance-Tests.cpp new file mode 100644 index 000000000..5d7cb3543 --- /dev/null +++ b/MMCore/unittest/CameraConformance-Tests.cpp @@ -0,0 +1,401 @@ +#include + +#include "DeviceBase.h" +#include "DeviceConformance/ConformanceTestConfig.h" +#include "MMCore.h" +#include "MMDeviceConstants.h" +#include "MockDeviceUtils.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +std::string GetTestStatus(const nlohmann::json& results, + const std::string& testName) { + for (const auto& t : results["tests"]) { + if (t["name"] == testName) + return t["status"].get(); + } + throw std::runtime_error("Test not found: " + testName); +} + +const mmcore::internal::ConformanceTestConfig shortTimeoutConfig{ + std::chrono::milliseconds(100), + std::chrono::milliseconds(50), +}; + +struct ConfigurableAsyncCamera : CCameraBase { + std::string name = "ConfigurableAsyncCamera"; + unsigned width = 64; + unsigned height = 64; + unsigned bytesPerPixel = 1; + unsigned nComponents = 1; + unsigned bitDepth = 8; + int binning = 1; + double exposure = 10.0; + + bool callPrepareForAcq = true; + bool checkPrepareForAcqReturn = true; + bool callAcqFinished = true; + bool clearCapturingBeforeAcqFinished = true; + bool checkInsertImageReturn = true; + bool failStartSequenceAcq = false; + + int Initialize() override { return DEVICE_OK; } + int Shutdown() override { return DEVICE_OK; } + bool Busy() override { return false; } + void GetName(char* buf) const override { + CDeviceUtils::CopyLimitedString(buf, name.c_str()); + } + + int SnapImage() override { + imgBuf_.assign( + static_cast(width) * height * bytesPerPixel, 0); + return DEVICE_OK; + } + const unsigned char* GetImageBuffer() override { + return imgBuf_.data(); + } + long GetImageBufferSize() const override { + return static_cast(width) * height * bytesPerPixel; + } + unsigned GetImageWidth() const override { return width; } + unsigned GetImageHeight() const override { return height; } + unsigned GetImageBytesPerPixel() const override { return bytesPerPixel; } + unsigned GetNumberOfComponents() const override { return nComponents; } + unsigned GetBitDepth() const override { return bitDepth; } + int GetBinning() const override { return binning; } + int SetBinning(int b) override { binning = b; return DEVICE_OK; } + void SetExposure(double e) override { exposure = e; } + double GetExposure() const override { return exposure; } + int SetROI(unsigned, unsigned, unsigned, unsigned) override { + return DEVICE_OK; + } + int GetROI(unsigned& x, unsigned& y, unsigned& w, unsigned& h) override { + x = 0; y = 0; w = width; h = height; + return DEVICE_OK; + } + int ClearROI() override { return DEVICE_OK; } + int IsExposureSequenceable(bool& seq) const override { + seq = false; + return DEVICE_OK; + } + + int StartSequenceAcquisition(long numImages, double, bool) override { + if (failStartSequenceAcq) + return DEVICE_ERR; + // Thread may be left over from previous (unstopped) run + if (thread_.joinable()) + thread_.join(); + if (callPrepareForAcq) { + int ret = GetCoreCallback()->PrepareForAcq(this); + if (checkPrepareForAcqReturn && ret != DEVICE_OK) + return ret; + } + { + std::lock_guard lk(mu_); + running_ = true; + stopRequested_ = false; + } + thread_ = std::thread([this, numImages] { + AcqThread(numImages); + }); + return DEVICE_OK; + } + + int StartSequenceAcquisition(double intervalMs) override { + return StartSequenceAcquisition(-1, intervalMs, false); + } + + ~ConfigurableAsyncCamera() { + { + std::lock_guard lk(mu_); + stopRequested_ = true; + } + cv_.notify_one(); + if (thread_.joinable()) + thread_.join(); + } + + int StopSequenceAcquisition() override { + { + std::lock_guard lk(mu_); + stopRequested_ = true; + } + cv_.notify_one(); + if (thread_.joinable()) + thread_.join(); + return DEVICE_OK; + } + + bool IsCapturing() override { + std::lock_guard lk(mu_); + return running_; + } + +private: + void AcqThread(long numImages) { + std::vector buf( + static_cast(width) * height * bytesPerPixel, 0); + long count = 0; + while (numImages < 0 || count < numImages) { + { + std::lock_guard lk(mu_); + if (stopRequested_) + break; + } + int ret = GetCoreCallback()->InsertImage(this, buf.data(), + width, height, bytesPerPixel, nComponents); + if (checkInsertImageReturn && ret != DEVICE_OK) + break; + ++count; + { + std::unique_lock lk(mu_); + cv_.wait_for(lk, std::chrono::microseconds(100), + [this] { return stopRequested_; }); + if (stopRequested_) + break; + } + } + if (clearCapturingBeforeAcqFinished) { + std::lock_guard lk(mu_); + running_ = false; + } + if (callAcqFinished) + GetCoreCallback()->AcqFinished(this, 0); + if (!clearCapturingBeforeAcqFinished) { + std::lock_guard lk(mu_); + running_ = false; + } + } + + bool running_ = false; + bool stopRequested_ = false; + std::mutex mu_; + std::condition_variable cv_; + std::thread thread_; + std::vector imgBuf_; +}; + +} // anonymous namespace + +TEST_CASE("Conformant camera passes all conformance tests", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); + for (const auto& test : results["tests"]) { + INFO("Test: " << test["name"].get()); + CHECK(test["status"].get() == "pass"); + } + CHECK(results["summary"]["passed"].get() == + results["summary"]["total"].get()); +} + +TEST_CASE("Missing PrepareForAcq is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.callPrepareForAcq = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-prepare-before-insert")); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "fail"); +} + +TEST_CASE("Missing AcqFinished is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.callAcqFinished = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-finished-after-count")); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "fail"); +} + +TEST_CASE("IsCapturing() true during AcqFinished is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.clearCapturingBeforeAcqFinished = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-finished-after-count")); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "fail"); +} + +TEST_CASE("Ignoring InsertImage error return is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.checkInsertImageReturn = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-finished-on-error-finite")); + CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "fail"); +} + +TEST_CASE("Camera that fails to start seq acq produces warning", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.failStartSequenceAcq = true; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse( + c.runDeviceConformanceTests("cam", "seq-basic")); + CHECK(GetTestStatus(results, "seq-basic") == "warning"); +} + +TEST_CASE("Dependent tests are skipped when dependency warns", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.failStartSequenceAcq = true; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); + CHECK(GetTestStatus(results, "seq-basic") == "warning"); + CHECK(GetTestStatus(results, "seq-prepare-before-insert") == "skipped"); + CHECK(GetTestStatus(results, "seq-finished-after-count") == "skipped"); + CHECK(GetTestStatus(results, "seq-finished-on-error-finite") == "skipped"); + CHECK(GetTestStatus(results, "seq-explicit-stop-finite") == "skipped"); + CHECK(GetTestStatus(results, "seq-explicit-stop-continuous") == "skipped"); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "warning"); + CHECK(results["summary"]["warnings"].get() == 2); + CHECK(results["summary"]["skipped"].get() == 8); +} + +TEST_CASE("deviceState contains camera settings", "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests("cam")); + CHECK(results["version"].get() == 3); + REQUIRE(results["deviceState"].is_object()); + + const auto& state = results["deviceState"]; + REQUIRE(state["properties"].is_object()); + REQUIRE(state["settings"].is_object()); + + const auto& settings = state["settings"]; + CHECK(settings["exposure"].get() == 10.0); + CHECK(settings["binning"].get() == 1); + CHECK(settings["imageWidth"].get() == 64); + CHECK(settings["imageHeight"].get() == 64); + CHECK(settings["bitDepth"].get() == 8); + CHECK(settings["roi"] == + nlohmann::json({{"x", 0}, {"y", 0}, {"width", 64}, {"height", 64}})); + CHECK(settings["exposureSequenceable"].get() == false); + CHECK(settings["multiROISupported"].get() == false); +} + +TEST_CASE("Ignoring PrepareForAcq error is detected by conformance test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + cam.checkPrepareForAcqReturn = false; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests( + "cam", "seq-prepare-error-propagated")); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "fail"); +} + +TEST_CASE("Conformant camera passes PrepareForAcq error propagation test", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto results = nlohmann::json::parse(c.runDeviceConformanceTests( + "cam", "seq-prepare-error-propagated")); + CHECK(GetTestStatus(results, "seq-prepare-error-propagated") == "pass"); +} + +TEST_CASE("getDeviceConformanceTests returns test list with dependencies", + "[CameraConformance]") { + ConfigurableAsyncCamera cam; + MockAdapterWithDevices adapter{{"cam", &cam}}; + CMMCore c; + c.setConformanceTestConfig(shortTimeoutConfig); + adapter.LoadIntoCore(c); + c.setCameraDevice("cam"); + + auto listing = nlohmann::json::parse(c.getDeviceConformanceTests("cam")); + CHECK(listing["version"].get() == 3); + CHECK(listing["deviceType"].get() == "Camera"); + REQUIRE(listing["tests"].is_array()); + REQUIRE(listing["tests"].size() > 0); + + // Every test has name and dependsOn + for (const auto& t : listing["tests"]) { + REQUIRE(t.contains("name")); + REQUIRE(t.contains("dependsOn")); + REQUIRE(t["dependsOn"].is_array()); + } + + // Check a specific dependency + bool foundPrepareBeforeInsert = false; + for (const auto& t : listing["tests"]) { + if (t["name"] == "seq-prepare-before-insert") { + foundPrepareBeforeInsert = true; + REQUIRE(t["dependsOn"].size() == 1); + CHECK(t["dependsOn"][0].get() == "seq-basic"); + } + } + CHECK(foundPrepareBeforeInsert); + + // Listing should not contain result fields + for (const auto& t : listing["tests"]) { + CHECK_FALSE(t.contains("status")); + CHECK_FALSE(t.contains("assertions")); + } + CHECK_FALSE(listing.contains("summary")); + CHECK_FALSE(listing.contains("timestamp")); +} diff --git a/MMCore/unittest/meson.build b/MMCore/unittest/meson.build index 7245e693f..8849bf78c 100644 --- a/MMCore/unittest/meson.build +++ b/MMCore/unittest/meson.build @@ -11,6 +11,7 @@ catch2_with_main_dep = dependency( mmcore_test_sources = files( 'APIError-Tests.cpp', + 'CameraConformance-Tests.cpp', 'CircularBuffer-Tests.cpp', 'CoreCreateDestroy-Tests.cpp', 'CoreProperties-Tests.cpp', @@ -36,10 +37,11 @@ mmcore_test_exe = executable( dependencies: [ mmdevice_dep, catch2_with_main_dep, + nlohmann_json_dep, ], cpp_args: [ '-D_CRT_SECURE_NO_WARNINGS', # TODO Eliminate the need ], ) -test('MMCore tests', mmcore_test_exe) +test('MMCore tests', mmcore_test_exe, timeout: 120)