Fix overdue state doesn't account for timeperiods in HA cluster

lib/checker/checkercomponent.cpp

@@ -56,6 +56,9 @@ void CheckerComponent::OnConfigLoaded()
 	Checkable::OnNextCheckChanged.connect([this](const Checkable::Ptr& checkable, const Value&) {
 		NextCheckChangedHandler(checkable);
 	});
+	Checkable::OnRescheduleCheck.connect([this](const Checkable::Ptr& checkable, double nextCheck) {
+		NextCheckChangedHandler(checkable, nextCheck);
+	});
 }
 
 void CheckerComponent::Start(bool runtimeCreated)
@@ -69,7 +72,7 @@ void CheckerComponent::Start(bool runtimeCreated)
 	m_Thread = std::thread([this]() { CheckThreadProc(); });
 
 	m_ResultTimer = Timer::Create();
-	m_ResultTimer->SetInterval(5);
+	m_ResultTimer->SetInterval(m_ResultTimerInterval);
 	m_ResultTimer->OnTimerExpired.connect([this](const Timer * const&) { ResultTimerHandler(); });
 	m_ResultTimer->Start();
 }
@@ -136,16 +139,13 @@ void CheckerComponent::CheckThreadProc()
 
 		bool forced = checkable->GetForceNextCheck();
 		bool check = true;
-		bool notifyNextCheck = false;
 		double nextCheck = -1;
 
 		if (!forced) {
 			if (!checkable->IsReachable(DependencyCheckExecution)) {
 				Log(LogNotice, "CheckerComponent")
 					<< "Skipping check for object '" << checkable->GetName() << "': Dependency failed.";
-
 				check = false;
-				notifyNextCheck = true;
 			}
 
 			Host::Ptr host;
@@ -182,7 +182,6 @@ void CheckerComponent::CheckThreadProc()
 						<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", nextCheck);
 
 					check = false;
-					notifyNextCheck = true;
 				}
 			}
 		}
@@ -201,23 +200,15 @@ void CheckerComponent::CheckThreadProc()
 				checkable->UpdateNextCheck();
 			}
 
-			if (notifyNextCheck) {
-				// Trigger update event for Icinga DB
-				Checkable::OnNextCheckUpdated(checkable);
-			}
-
 			lock.lock();
 
 			continue;
 		}
 
-
-		csi = GetCheckableScheduleInfo(checkable);
-
 		Log(LogDebug, "CheckerComponent")
 			<< "Scheduling info for checkable '" << checkable->GetName() << "' ("
 			<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", checkable->GetNextCheck()) << "): Object '"
-			<< csi.Object->GetName() << "', Next Check: "
+			<< checkable->GetName() << "', Next Check: "
 			<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", csi.NextCheck)
 			<< " (" << std::fixed << std::setprecision(0) << csi.NextCheck << ").";
 
@@ -241,16 +232,16 @@ void CheckerComponent::CheckThreadProc()
 		 */
 		CheckerComponent::Ptr checkComponent(this);
 
-		Utility::QueueAsyncCallback([this, checkComponent, checkable]() { ExecuteCheckHelper(checkable); });
+		Utility::QueueAsyncCallback([this, checkComponent, checkable, csi]() { ExecuteCheckHelper(checkable, csi); });
 
 		lock.lock();
 	}
 }
 
-void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
+void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable, const CheckableScheduleInfo& csi)
 {
 	try {
-		checkable->ExecuteCheck(m_WaitGroup);
+		checkable->ExecuteCheck(m_WaitGroup, csi.NextCheck);
 	} catch (const std::exception& ex) {
 		CheckResult::Ptr cr = new CheckResult();
 		cr->SetState(ServiceUnknown);
@@ -342,7 +333,7 @@ CheckableScheduleInfo CheckerComponent::GetCheckableScheduleInfo(const Checkable
 	return csi;
 }
 
-void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
+void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable, double nextCheck)
 {
 	std::unique_lock<std::mutex> lock(m_Mutex);
 
@@ -357,7 +348,13 @@ void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 
 	idx.erase(checkable);
 
-	CheckableScheduleInfo csi = GetCheckableScheduleInfo(checkable);
+	CheckableScheduleInfo csi;
+	if (nextCheck < 0) {
+		csi = GetCheckableScheduleInfo(checkable);
+	} else {
+		csi.NextCheck = nextCheck;
+		csi.Object = checkable;
+	}
 	idx.insert(csi);
 
 	m_CV.notify_all();
@@ -376,3 +373,18 @@ unsigned long CheckerComponent::GetPendingCheckables()
 
 	return m_PendingCheckables.size();
 }
+
+/**
+ * Sets the interval in seconds for the result timer.
+ *
+ * The result timer periodically logs the number of pending and idle checkables
+ * as well as the checks per second rate. The default interval is 5 seconds.
+ *
+ * Note, this method must be called before the component is started to have an effect on the timer.
+ *
+ * @param interval Interval in seconds for the result timer.
+ */
+void CheckerComponent::SetResultTimerInterval(double interval)
+{
+	m_ResultTimerInterval = interval;
+}

lib/checker/checkercomponent.hpp

@@ -27,21 +27,25 @@ struct CheckableScheduleInfo
 {
 	Checkable::Ptr Object;
 	double NextCheck;
-};
-
-/**
- * @ingroup checker
- */
-struct CheckableNextCheckExtractor
-{
-	typedef double result_type;
 
 	/**
-	 * @threadsafety Always.
+	 * Get the index value for ordering in the multi-index container.
+	 *
+	 * This function returns a very large value for checkables that have a running check, effectively pushing
+	 * them to the end of the ordering. This ensures that checkables with running checks are not prioritized
+	 * for scheduling ahead of others. Rescheduling of such checkables is unnecessary because the checkable
+	 * is going to reject this anyway if it notices that a check is already running, so avoiding unnecessary
+	 * CPU load. Once the running check is finished, the checkable will be re-inserted into the set with its
+	 * actual next check time as the index value.
+	 *
+	 * @return The index value for ordering in the multi-index container.
 	 */
-	double operator()(const CheckableScheduleInfo& csi)
+	[[nodiscard]] double Index() const
 	{
-		return csi.NextCheck;
+		if (Object->HasRunningCheck()) {
+			return std::numeric_limits<double>::max();
+		}
+		return NextCheck;
 	}
 };
 
@@ -58,7 +62,7 @@ class CheckerComponent final : public ObjectImpl<CheckerComponent>
 		CheckableScheduleInfo,
 		boost::multi_index::indexed_by<
 			boost::multi_index::ordered_unique<boost::multi_index::member<CheckableScheduleInfo, Checkable::Ptr, &CheckableScheduleInfo::Object> >,
-			boost::multi_index::ordered_non_unique<CheckableNextCheckExtractor>
+			boost::multi_index::ordered_non_unique<boost::multi_index::const_mem_fun<CheckableScheduleInfo, double, &CheckableScheduleInfo::Index>>
 		>
 	> CheckableSet;
 
@@ -70,12 +74,16 @@ class CheckerComponent final : public ObjectImpl<CheckerComponent>
 	unsigned long GetIdleCheckables();
 	unsigned long GetPendingCheckables();
 
+	void SetResultTimerInterval(double interval);
+
 private:
 	std::mutex m_Mutex;
 	std::condition_variable m_CV;
 	bool m_Stopped{false};
 	std::thread m_Thread;
 
+	double m_ResultTimerInterval{5.0}; // Interval in seconds for the result timer.
+
 	CheckableSet m_IdleCheckables;
 	CheckableSet m_PendingCheckables;
 
@@ -85,14 +93,10 @@ class CheckerComponent final : public ObjectImpl<CheckerComponent>
 	void CheckThreadProc();
 	void ResultTimerHandler();
 
-	void ExecuteCheckHelper(const Checkable::Ptr& checkable);
-
-	void AdjustCheckTimer();
+	void ExecuteCheckHelper(const Checkable::Ptr& checkable, const CheckableScheduleInfo& csi);
 
 	void ObjectHandler(const ConfigObject::Ptr& object);
-	void NextCheckChangedHandler(const Checkable::Ptr& checkable);
-
-	void RescheduleCheckTimer();
+	void NextCheckChangedHandler(const Checkable::Ptr& checkable, double nextCheck = -1);
 
 	static CheckableScheduleInfo GetCheckableScheduleInfo(const Checkable::Ptr& checkable);
 };

lib/db_ido/dbevents.cpp

@@ -41,7 +41,7 @@ void DbEvents::StaticInitialize()
 		DbEvents::RemoveAcknowledgement(checkable);
 	});
 
-	Checkable::OnNextCheckUpdated.connect([](const Checkable::Ptr& checkable) { NextCheckUpdatedHandler(checkable); });
+	Checkable::OnNextCheckChanged.connect([](const Checkable::Ptr& checkable, const Value&) { NextCheckChangedHandler(checkable); });
 	Checkable::OnFlappingChanged.connect([](const Checkable::Ptr& checkable, const Value&) { FlappingChangedHandler(checkable); });
 	Checkable::OnNotificationSentToAllUsers.connect([](const Notification::Ptr& notification, const Checkable::Ptr& checkable,
 		const std::set<User::Ptr>&, const NotificationType&, const CheckResult::Ptr&, const String&, const String&,
@@ -119,7 +119,7 @@ void DbEvents::StaticInitialize()
 }
 
 /* check events */
-void DbEvents::NextCheckUpdatedHandler(const Checkable::Ptr& checkable)
+void DbEvents::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
 	Host::Ptr host;
 	Service::Ptr service;

lib/db_ido/dbevents.hpp

@@ -53,7 +53,7 @@ class DbEvents
 	static void AddLogHistory(const Checkable::Ptr& checkable, const String& buffer, LogEntryType type);
 
 	/* Status */
-	static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable);
+	static void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 	static void FlappingChangedHandler(const Checkable::Ptr& checkable);
 	static void LastNotificationChangedHandler(const Notification::Ptr& notification, const Checkable::Ptr& checkable);
 

lib/icinga/apiactions.cpp

@@ -166,9 +166,6 @@ Dictionary::Ptr ApiActions::RescheduleCheck(
 
 	checkable->SetNextCheck(nextCheck);
 
-	/* trigger update event for DB IDO */
-	Checkable::OnNextCheckUpdated(checkable);
-
 	return ApiActions::CreateResult(200, "Successfully rescheduled check for object '" + checkable->GetName() + "'.");
 }