Merge pull request #321 from github/ccr-setup

Add Support for Configuring CCR Setup

Author: jherns

README.md

@@ -142,6 +142,11 @@ To run ES 5 and ES 8:
 docker compose --project-directory docker --profile all up
 ```
 
+To run in ES8 cross cluster replication mode:
+```
+script/setup-ccr up "{non-production license}"
+```
+
 To run only ES 8:
 ```
 docker compose --project-directory docker --profile es8 up

docker/compose.yaml

@@ -4,14 +4,15 @@ services:
   elasticsearch8.13:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.13.2
     container_name: es8.13
-    profiles: ["es8", "all"]
+    profiles: ["es8", "ccr", "all"]
     environment:
       - cluster.name=elastomer8.13
       - bootstrap.memory_lock=true
       - discovery.type=single-node
       - xpack.security.enabled=false
       - xpack.watcher.enabled=false
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - node.roles=[master,data,remote_cluster_client]
     ulimits:
       memlock:
         soft: -1
@@ -28,6 +29,33 @@ services:
     ports:
       - 127.0.0.1:${ES_8_PORT:-9208}:9200
 
+  elasticsearchFollower:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.2
+    container_name: es-follow
+    profiles: ["ccr"]
+    environment:
+      - cluster.name=es-follow
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - node.roles=[master,data,remote_cluster_client]
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 2g
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - ./elasticsearch-follow.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - 127.0.0.1:${ES_8_PORT:-9209}:9201
+
   elasticsearch5.6:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
     container_name: es5.6

docker/elasticsearch-follow.yml

@@ -0,0 +1,14 @@
+cluster.name: "es-follow"
+
+network.host: 0.0.0.0
+
+path:
+  data: /usr/share/elasticsearch/data
+  logs: /usr/share/elasticsearch/logs
+  repo: /usr/share/elasticsearch/repos
+
+transport.port: 9301
+http.port: 9201
+remote_cluster.port: 9444
+http.max_content_length: 50mb
+ingest.geoip.downloader.enabled: false

docker/elasticsearch8plus.yml

@@ -9,5 +9,6 @@ path:
 
 transport.port: 9300
 http.port: 9200
+remote_cluster.port: 9443
 http.max_content_length: 50mb
 ingest.geoip.downloader.enabled: false

script/setup-ccr

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Function to display help
+show_help() {
+  echo "Usage: $0 [option] [license]"
+  echo "Options:"
+  echo "  up      - Start the clusters and apply the license"
+  echo "  down    - Shut down the clusters"
+  echo "  help    - Display this help message"
+}
+
+# Function to apply the license to a cluster
+apply_license() {
+  local port=$1
+  local license="$2"
+  local response_file=$(mktemp)
+  local http_code
+  http_code=$(curl -s -o "$response_file" -w "%{http_code}" -X PUT "http://localhost:$port/_license?pretty" -H "Content-Type: application/json" -d "$license")
+  
+  if [ "$http_code" -ne 200 ]; then
+    echo "Failed to apply license to cluster on port $port. HTTP status code: $http_code"
+    echo "Error response: $(cat "$response_file")"
+    rm "$response_file"
+    exit 1
+  fi
+}
+
+# Function to shut down the clusters
+shutdown_clusters() {
+  docker compose --project-directory docker --profile ccr down
+  echo "Clusters shut down."
+}
+
+# Check for options
+case "$1" in
+  up)
+
+    # Get the directory of the current script
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+    # Start the clusters
+    docker compose --project-directory docker --profile ccr up -d
+
+    # Wait for both clusters to be online
+    export ES_PORT=9208
+    "$SCRIPT_DIR/poll-for-es"
+    export ES_PORT=9209
+    "$SCRIPT_DIR/poll-for-es"
+
+    # Apply the license to both clusters
+    LICENSE=$2
+    if [ -z "$LICENSE" ]; then
+      echo "License key is required as the second argument."
+      exit 1
+    fi
+
+    echo "Applying license to cluster on port 9208..."
+    apply_license 9208 "$LICENSE"
+    echo "Applying license to cluster on port 9209..."
+    apply_license 9209 "$LICENSE"
+    echo "License applied to both clusters."
+
+    # Set up the remote connection between the clusters
+    curl -X PUT "http://localhost:9209/_cluster/settings" -H "Content-Type: application/json" -d '{
+      "persistent": {
+        "cluster": {
+          "remote": {
+            "cluster_one": {
+              "seeds": ["es8.13:9300"]
+            }
+          }
+        }
+      }
+    }'
+
+    echo "Clusters setup completed."
+    ;;
+  down)
+    shutdown_clusters
+    ;;
+  help)
+    show_help
+    ;;
+  *)
+    echo "Invalid option: $1"
+    show_help
+    exit 1
+    ;;
+esac