fix(opensearch): enforce ES/OS endpoint separation at the OS index-creation chokepoint (#36419)

dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java

@@ -1008,11 +1008,31 @@ private void bootstrapAndPointES(final String workingName, final String liveName
     private void bootstrapAndPointOS(final String workingName, final String liveName)
             throws DotDataException {
 
+        // Separation gate (issue #36419): OS must be a SEPARATE cluster from ES. Config-only,
+        // no network I/O, so it runs before the connection gate. Placing it at this single
+        // chokepoint closes the window where the empty-DB starter-load path created .os indices
+        // before InitServlet's later validateIndexingConfig() caught the ES==OS overlap. On
+        // overlap we halt the migration (ES-only) and skip OS bootstrap entirely.
+        // Phase-aware: in Phase 3 (ES decommissioned, ES_ENDPOINTS not required) the check is
+        // skipped inside endpointsAreSeparate(), so this branch never fires there and the
+        // haltMigration() fallback below only ever runs in dual-write phases where ES is live.
+        if (!IndexStartupValidator.endpointsAreSeparate()) {
+            Logger.warn(this.getClass(),
+                    "Skipping OpenSearch index bootstrap (working=" + workingName
+                    + ", live=" + liveName + "): OS migration configuration rejected"
+                    + " (see the preceding error for the cause — e.g. ES/OS endpoint overlap or"
+                    + " an unresolved OS config). Migration halted (now ES-only).");
+            haltMigration();
+            return;
+        }
+
         // Connection gate (issue #36244): verify OS reachability BEFORE creating OS indices.
-        // This is the single chokepoint for all OS index creation (fresh-install bootstrap and
+        // This is the single chokepoint for OS working/live index bootstrap (fresh-install and
         // migration catchup), so both startup paths — populated-DB (InitServlet) and empty-DB
         // (Task00004LoadStarter) — pass through the same phase-aware gate instead of failing
         // late and opaquely with a transport exception deep inside createContentIndex.
+        // (Reindex-slot creation via initAndPointReindex does NOT pass through here — see the
+        // runtime phase-flip caveat in PR #36421.)
         //
         // operationsOS.indexAPI() is the OS-specific IndexAPI, so the gate always probes OS
         // regardless of the current read provider (in Phase 1 the read provider is ES). The

dotCMS/src/main/java/com/dotcms/content/index/opensearch/IndexStartupValidator.java

@@ -87,7 +87,15 @@ public void validate() {
         final OSClientConfig config = resolveConfig();
         logConfigSummary(config);
         validateOSVersion();
-        validateEndpointSeparation(config);
+        if (IndexConfigHelper.isMigrationComplete()) {
+            // Phase 3: ES is decommissioned and ES_ENDPOINTS is no longer required, so the
+            // resolved ES side would just be the localhost:9200 default — comparing it against
+            // OS would false-positive on a legitimate config. Skip the separation check.
+            Logger.info(this, "Endpoint separation check skipped: PHASE_3_OPENSEARCH_ONLY"
+                    + " — ES is decommissioned and ES_ENDPOINTS is not required.");
+            return;
+        }
+        assertEndpointsSeparate(config);
     }
 
     // -------------------------------------------------------------------------
@@ -204,19 +212,17 @@ private void validateOSVersion() {
     // -------------------------------------------------------------------------
 
     /**
-     * Resolves the effective ES and OS endpoint sets and asserts they do not share
-     * any {@code host:port} pair.
-     *
-     * <p>The OS side comes from the already-resolved {@link OSClientConfig#endpoints()} (the very
-     * endpoints the client is built with), and the ES side is read from config; both are passed
-     * through the same {@link #normalizeEndpoint} path so the comparison is consistent.</p>
+     * Asserts that the ES and OS clients do not share any {@code host:port}. Shared by the
+     * startup validator ({@link #validate()}) and the config-only OS index-creation gate
+     * ({@link #endpointsAreSeparate()}) so both enforce separation with identical logic.
+     * On success it logs the compared sets so the "passed" line is proof of what was compared.
      *
      * <p><strong>Best-effort:</strong> two configs that refer to the same host via different forms
      * (e.g. {@code "127.0.0.1"} vs {@code "localhost"}) will not be detected as overlapping.</p>
      *
      * @throws DotRuntimeException if at least one endpoint is common to both clients
      */
-    private void validateEndpointSeparation(final OSClientConfig config) {
+    static void assertEndpointsSeparate(final OSClientConfig config) {
         final Set<String> esEndpoints = resolveESEndpoints();
         final Set<String> osEndpoints = config.endpoints().stream()
                 .map(IndexStartupValidator::normalizeEndpoint)
@@ -232,10 +238,54 @@ private void validateEndpointSeparation(final OSClientConfig config) {
                     + " — OS endpoints: " + osEndpoints
                     + ". Set OS_ENDPOINTS to a separate OpenSearch instance.");
         }
-        Logger.info(this, "Endpoint separation check passed."
+        Logger.info(IndexStartupValidator.class, "Endpoint separation check passed."
                 + " ES: " + esEndpoints + " — OS: " + osEndpoints);
     }
 
+    /**
+     * Config-only endpoint-separation gate for the OS index-creation chokepoint
+     * ({@code ContentletIndexAPIImpl.bootstrapAndPointOS}, issue #36419). Unlike
+     * {@link #validate()} this performs no network I/O — it only compares the resolved ES and OS
+     * endpoints — so it is cheap enough to run before every OS bootstrap on any startup path
+     * (empty-DB starter-load or populated-DB InitServlet), closing the window where the
+     * starter-load path created {@code .os} indices before the late startup validation ran.
+     *
+     * <p><strong>Phase-aware:</strong> in {@code PHASE_3_OPENSEARCH_ONLY} the check is skipped
+     * (returns {@code true}) — ES is decommissioned and {@code ES_ENDPOINTS} is no longer
+     * required, so the resolved ES side would just be the {@code localhost:9200} default and
+     * comparing it against OS would false-positive on a legitimate configuration.</p>
+     *
+     * @return {@code true} when ES and OS point to distinct clusters (safe to create OS indices);
+     *         {@code false} when they overlap (caller must skip the OS bootstrap and halt the migration)
+     */
+    public static boolean endpointsAreSeparate() {
+        if (IndexConfigHelper.isMigrationComplete()) {
+            Logger.info(IndexStartupValidator.class,
+                    "Endpoint separation check skipped: PHASE_3_OPENSEARCH_ONLY"
+                    + " — ES is decommissioned and ES_ENDPOINTS is not required.");
+            return true;
+        }
+        final OSClientConfig config;
+        try {
+            config = ConfigurableOpenSearchProvider.configFromProperties();
+        } catch (Exception e) {
+            // Config could not be resolved — this is NOT an ES/OS overlap. Log it distinctly so the
+            // operator is not sent chasing a same-endpoint misconfiguration that does not exist.
+            Logger.error(IndexStartupValidator.class,
+                    "Cannot resolve OpenSearch configuration for the endpoint-separation check: "
+                    + e.getMessage());
+            return false;
+        }
+        try {
+            assertEndpointsSeparate(config);
+            return true;
+        } catch (DotRuntimeException e) {
+            // The overlap message (with the actionable "Set OS_ENDPOINTS to a separate instance").
+            Logger.error(IndexStartupValidator.class, e.getMessage());
+            return false;
+        }
+    }
+
     /**
      * Resolves ES endpoints from configuration, passing them through the same
      * {@link #normalizeEndpoint} path used for the OS side so both can be compared consistently.

dotCMS/src/test/java/com/dotcms/content/index/opensearch/IndexStartupValidatorTest.java

@@ -0,0 +1,124 @@
+package com.dotcms.content.index.opensearch;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.dotcms.content.index.IndexConfigHelper;
+import com.dotmarketing.exception.DotRuntimeException;
+import com.dotmarketing.util.Config;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link IndexStartupValidator} endpoint-separation logic.
+ *
+ * <p>Covers the config-only separation gate ({@link IndexStartupValidator#endpointsAreSeparate()})
+ * used at the OS index-creation chokepoint (issue #36419): OS must point to a cluster separate
+ * from ES, otherwise the {@code .os} shadow indices must not be created.</p>
+ */
+public class IndexStartupValidatorTest {
+
+    @After
+    public void clearEndpoints() {
+        Config.setProperty("ES_ENDPOINTS", null);
+        Config.setProperty("OS_ENDPOINTS", null);
+        Config.setProperty(IndexConfigHelper.MigrationPhase.FLAG_KEY, 0);
+    }
+
+    /**
+     * Given Scenario: ES and OS are configured with the same address.
+     * Expected Result: endpointsAreSeparate() returns false — OS bootstrap must be skipped.
+     */
+    @Test
+    public void test_endpointsAreSeparate_sameAddress_returnsFalse() {
+        Config.setProperty("ES_ENDPOINTS", new String[]{"https://localhost:9201"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{"https://localhost:9201"});
+
+        assertFalse(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: ES and OS point at different ports on the same host.
+     * Expected Result: endpointsAreSeparate() returns true — safe to create OS indices.
+     */
+    @Test
+    public void test_endpointsAreSeparate_differentAddress_returnsTrue() {
+        Config.setProperty("ES_ENDPOINTS", new String[]{"https://localhost:9200"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{"https://localhost:9201"});
+
+        assertTrue(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: same address, but written with different scheme/formatting.
+     * Expected Result: overlap is still detected (normalized to host:port) — returns false.
+     */
+    @Test
+    public void test_endpointsAreSeparate_sameHostPortDifferentScheme_returnsFalse() {
+        Config.setProperty("ES_ENDPOINTS", new String[]{"http://localhost:9201"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{"https://localhost:9201"});
+
+        assertFalse(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: multi-endpoint ES [a,b] and OS [b,c] share one endpoint.
+     * Expected Result: partial overlap is detected — returns false.
+     */
+    @Test
+    public void test_endpointsAreSeparate_multiEndpointPartialOverlap_returnsFalse() {
+        Config.setProperty("ES_ENDPOINTS",
+                new String[]{"https://es-a:9200", "https://shared:9200"});
+        Config.setProperty("OS_ENDPOINTS",
+                new String[]{"https://shared:9200", "https://os-c:9200"});
+
+        assertFalse(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: OS config cannot be resolved (blank endpoint trips OSClientConfig validation).
+     * Expected Result: endpointsAreSeparate() fails closed — returns false — rather than throwing
+     *                  out of the OS bootstrap gate.
+     */
+    @Test
+    public void test_endpointsAreSeparate_unresolvableOsConfig_returnsFalse() {
+        Config.setProperty("ES_ENDPOINTS", new String[]{"https://localhost:9200"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{""});
+
+        assertFalse(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: Phase 3 (OPENSEARCH_ONLY) with ES and OS resolving to the same address —
+     *                 legitimate, since ES is decommissioned and ES_ENDPOINTS is not required.
+     * Expected Result: the separation check is skipped — endpointsAreSeparate() returns true.
+     */
+    @Test
+    public void test_endpointsAreSeparate_phase3_checkSkipped_returnsTrue() {
+        Config.setProperty(IndexConfigHelper.MigrationPhase.FLAG_KEY, 3);
+        Config.setProperty("ES_ENDPOINTS", new String[]{"https://localhost:9201"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{"https://localhost:9201"});
+
+        assertTrue(IndexStartupValidator.endpointsAreSeparate());
+    }
+
+    /**
+     * Given Scenario: ES and OS share the same host:port.
+     * Expected Result: assertEndpointsSeparate throws DotRuntimeException naming the overlap.
+     */
+    @Test
+    public void test_assertEndpointsSeparate_overlap_throws() {
+        Config.setProperty("ES_ENDPOINTS", new String[]{"https://localhost:9201"});
+        Config.setProperty("OS_ENDPOINTS", new String[]{"https://localhost:9201"});
+
+        try {
+            IndexStartupValidator.assertEndpointsSeparate(
+                    ConfigurableOpenSearchProvider.configFromProperties());
+            fail("Expected DotRuntimeException for overlapping ES/OS endpoints");
+        } catch (final DotRuntimeException e) {
+            assertTrue("message should name the overlap",
+                    e.getMessage().contains("point to the same endpoint(s)"));
+        }
+    }
+}