Address Entity-table limitations

graylog2-server/src/main/java/org/graylog2/bindings/ServerBindings.java

@@ -46,13 +46,16 @@
 import org.graylog2.cluster.leader.FakeLeaderElectionModule;
 import org.graylog2.cluster.leader.LeaderElectionModule;
 import org.graylog2.cluster.lock.LockServiceModule;
+import org.graylog2.database.filtering.ComputedFieldProvider;
+import org.graylog2.database.filtering.ComputedFieldRegistry;
 import org.graylog2.grok.GrokModule;
 import org.graylog2.grok.GrokPatternRegistry;
 import org.graylog2.indexer.fieldtypes.FieldTypesModule;
 import org.graylog2.indexer.healing.FixDeflectorByDeleteJob;
 import org.graylog2.indexer.healing.FixDeflectorByMoveJob;
 import org.graylog2.indexer.indices.jobs.IndexSetCleanupJob;
 import org.graylog2.inputs.InputEventListener;
+import org.graylog2.inputs.InputRuntimeStatusProvider;
 import org.graylog2.inputs.InputStateListener;
 import org.graylog2.inputs.PersistedInputsImpl;
 import org.graylog2.lookup.LookupModule;
@@ -218,6 +221,10 @@ private void bindInterfaces() {
         OptionalBinder.newOptionalBinder(binder(), TrafficUpdater.class).setDefault().to(TrafficCounterService.class).asEagerSingleton();
 
         Multibinder.newSetBinder(binder(), PluggableEntityHandler.class);
+
+        bind(ComputedFieldRegistry.class).in(Scopes.SINGLETON);
+        final Multibinder<ComputedFieldProvider> computedFieldProviders = Multibinder.newSetBinder(binder(), ComputedFieldProvider.class);
+        computedFieldProviders.addBinding().to(InputRuntimeStatusProvider.class);
     }
 
     private void bindDynamicFeatures() {

graylog2-server/src/main/java/org/graylog2/database/filtering/ComputedFieldProvider.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.database.filtering;
+
+import java.util.Set;
+
+/**
+ * Provides filtering support for computed/runtime fields that don't exist in the database.
+ * <p>
+ * Implementations return matching entity IDs based on filter values. The filtering system
+ * injects these IDs into the database query using MongoDB's $in operator, allowing efficient
+ * filtering on runtime/computed values alongside regular database fields.
+ * </p>
+ * <p>
+ * <b>Example Usage:</b>
+ * </p>
+ * <pre>
+ * // Provider implementation for filtering inputs by runtime status
+ * public class InputRuntimeStatusProvider implements ComputedFieldProvider {
+ *     &#64;Override
+ *     public Set&lt;String&gt; getMatchingIds(String filterValue) {
+ *         // Query all cluster nodes for inputs with status = filterValue
+ *         // Return set of matching input IDs
+ *         return matchingInputIds;
+ *     }
+ *
+ *     &#64;Override
+ *     public String getFieldName() {
+ *         return "runtime_status";
+ *     }
+ * }
+ * </pre>
+ * <p>
+ * <b>Integration Flow:</b>
+ * </p>
+ * <ol>
+ *   <li>User applies filter: runtime_status:FAILED</li>
+ *   <li>Filter parser detects "runtime_status" is a computed field</li>
+ *   <li>Calls InputRuntimeStatusProvider.getMatchingIds("FAILED")</li>
+ *   <li>Provider queries all cluster nodes for failed inputs</li>
+ *   <li>Returns Set&lt;String&gt; of matching input IDs</li>
+ *   <li>Injected into MongoDB query: { _id: { $in: [...ids] } }</li>
+ *   <li>MongoDB returns only matching inputs</li>
+ * </ol>
+ * <p>
+ * Providers should be registered with Guice using a Multibinder:
+ * </p>
+ * <pre>
+ * Multibinder&lt;ComputedFieldProvider&gt; binder =
+ *     Multibinder.newSetBinder(binder(), ComputedFieldProvider.class);
+ * binder.addBinding().to(InputRuntimeStatusProvider.class);
+ * </pre>
+ */
+public interface ComputedFieldProvider {
+    /**
+     * Returns the set of entity IDs that match the given filter value.
+     * <p>
+     * This method should:
+     * </p>
+     * <ul>
+     *   <li>Query runtime/computed data sources (registries, caches, external APIs, etc.)</li>
+     *   <li>Filter entities based on the provided filter value</li>
+     *   <li>Return the IDs of entities that match the criteria</li>
+     *   <li>Return an empty set if no entities match or if the filter value is invalid</li>
+     * </ul>
+     * <p>
+     * <b>Performance Considerations:</b>
+     * </p>
+     * <ul>
+     *   <li>This method is called during query construction, so it should be reasonably fast</li>
+     *   <li>For cluster-wide data, consider parallel queries to multiple nodes</li>
+     *   <li>Consider caching if the computed values change infrequently</li>
+     * </ul>
+     *
+     * @param filterValue The value being filtered on (e.g., "FAILED", "RUNNING")
+     * @param authToken Optional authentication token for cluster-wide operations. May be null.
+     * @return Set of entity IDs that match the filter value. Empty set if no matches.
+     */
+    Set<String> getMatchingIds(String filterValue, String authToken);
+
+    /**
+     * Returns the field name this provider handles.
+     * <p>
+     * This name should match the field name used in filter expressions and
+     * entity attribute definitions.
+     * </p>
+     * <p>
+     * <b>Example:</b> "runtime_status", "computed_health", "aggregated_state"
+     * </p>
+     *
+     * @return The field name this provider handles (e.g., "runtime_status")
+     */
+    String getFieldName();
+}

graylog2-server/src/main/java/org/graylog2/database/filtering/ComputedFieldRegistry.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.database.filtering;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Registry of all computed field providers in the system.
+ * <p>
+ * This registry maintains a mapping of field names to their corresponding
+ * {@link ComputedFieldProvider} implementations. It is used by the filtering
+ * system to determine whether a field is computed and to retrieve the appropriate
+ * provider for filtering operations.
+ * </p>
+ * <p>
+ * Providers are automatically discovered and registered through Guice's
+ * Multibinder mechanism. To add a new computed field provider:
+ * </p>
+ * <ol>
+ *   <li>Create a class that implements {@link ComputedFieldProvider}</li>
+ *   <li>Register it with Guice using a Multibinder</li>
+ *   <li>The registry will automatically discover and include it</li>
+ * </ol>
+ * <p>
+ * <b>Example Guice Module Configuration:</b>
+ * </p>
+ * <pre>
+ * Multibinder&lt;ComputedFieldProvider&gt; computedFieldBinder =
+ *     Multibinder.newSetBinder(binder(), ComputedFieldProvider.class);
+ * computedFieldBinder.addBinding().to(InputRuntimeStatusProvider.class);
+ * computedFieldBinder.addBinding().to(AnotherComputedFieldProvider.class);
+ * </pre>
+ * <p>
+ * <b>Thread Safety:</b> This class is thread-safe. The internal map is immutable
+ * after construction.
+ * </p>
+ */
+@Singleton
+public class ComputedFieldRegistry {
+    private final Map<String, ComputedFieldProvider> providers;
+
+    /**
+     * Constructs the registry by indexing all available providers by their field names.
+     * <p>
+     * This constructor is called by Guice with all registered {@link ComputedFieldProvider}
+     * implementations injected via Multibinder.
+     * </p>
+     *
+     * @param providers Set of all registered computed field providers
+     * @throws IllegalStateException if multiple providers claim the same field name
+     */
+    @Inject
+    public ComputedFieldRegistry(Set<ComputedFieldProvider> providers) {
+        this.providers = providers.stream()
+                .collect(Collectors.toMap(
+                        ComputedFieldProvider::getFieldName,
+                        Function.identity(),
+                        (p1, p2) -> {
+                            throw new IllegalStateException(
+                                    "Multiple ComputedFieldProviders registered for field: " +
+                                            p1.getFieldName() + " (" +
+                                            p1.getClass().getName() + " and " +
+                                            p2.getClass().getName() + ")"
+                            );
+                        }
+                ));
+    }
+
+    /**
+     * Retrieves the provider for the specified field name.
+     * <p>
+     * <b>Example:</b>
+     * </p>
+     * <pre>
+     * Optional&lt;ComputedFieldProvider&gt; provider = registry.getProvider("runtime_status");
+     * if (provider.isPresent()) {
+     *     Set&lt;String&gt; matchingIds = provider.get().getMatchingIds("RUNNING");
+     *     // Use matchingIds in query...
+     * }
+     * </pre>
+     *
+     * @param fieldName The name of the computed field
+     * @return Optional containing the provider if one exists for the field, empty otherwise
+     */
+    public Optional<ComputedFieldProvider> getProvider(String fieldName) {
+        return Optional.ofNullable(providers.get(fieldName));
+    }
+
+    /**
+     * Checks whether a field name corresponds to a computed field.
+     * <p>
+     * This is a convenience method used to quickly determine if a field should
+     * be handled by a computed field provider or by the standard database query logic.
+     * </p>
+     * <p>
+     * <b>Example:</b>
+     * </p>
+     * <pre>
+     * if (registry.isComputedField("runtime_status")) {
+     *     // Handle as computed field
+     * } else {
+     *     // Handle as database field
+     * }
+     * </pre>
+     *
+     * @param fieldName The name of the field to check
+     * @return true if a provider exists for this field, false otherwise
+     */
+    public boolean isComputedField(String fieldName) {
+        return providers.containsKey(fieldName);
+    }
+}