This documentation contains work-in-progress information for future Elastic Stack and Cloud releases. Use the version selector to view supported release docs. It also contains some Elastic Cloud serverless information. Check out our serverless docs for more details.

« Tutorial: Disaster recovery based on bi-directional cross-cluster replication Failover when clusterA is down »

› › ›

Initial setup

edit

Initial setup

edit

Set up a remote cluster on both clusters.

resp = client.cluster.put_settings(
    persistent={
        "cluster": {
            "remote": {
                "clusterB": {
                    "mode": "proxy",
                    "skip_unavailable": True,
                    "server_name": "clusterb.es.region-b.gcp.elastic-cloud.com",
                    "proxy_socket_connections": 18,
                    "proxy_address": "clusterb.es.region-b.gcp.elastic-cloud.com:9400"
                }
            }
        }
    },
)
print(resp)

resp1 = client.cluster.put_settings(
    persistent={
        "cluster": {
            "remote": {
                "clusterA": {
                    "mode": "proxy",
                    "skip_unavailable": True,
                    "server_name": "clustera.es.region-a.gcp.elastic-cloud.com",
                    "proxy_socket_connections": 18,
                    "proxy_address": "clustera.es.region-a.gcp.elastic-cloud.com:9400"
                }
            }
        }
    },
)
print(resp1)

response = client.cluster.put_settings(
  body: {
    persistent: {
      cluster: {
        remote: {
          "clusterB": {
            mode: 'proxy',
            skip_unavailable: true,
            server_name: 'clusterb.es.region-b.gcp.elastic-cloud.com',
            proxy_socket_connections: 18,
            proxy_address: 'clusterb.es.region-b.gcp.elastic-cloud.com:9400'
          }
        }
      }
    }
  }
)
puts response

response = client.cluster.put_settings(
  body: {
    persistent: {
      cluster: {
        remote: {
          "clusterA": {
            mode: 'proxy',
            skip_unavailable: true,
            server_name: 'clustera.es.region-a.gcp.elastic-cloud.com',
            proxy_socket_connections: 18,
            proxy_address: 'clustera.es.region-a.gcp.elastic-cloud.com:9400'
          }
        }
      }
    }
  }
)
puts response

const response = await client.cluster.putSettings({
  persistent: {
    cluster: {
      remote: {
        clusterB: {
          mode: "proxy",
          skip_unavailable: true,
          server_name: "clusterb.es.region-b.gcp.elastic-cloud.com",
          proxy_socket_connections: 18,
          proxy_address: "clusterb.es.region-b.gcp.elastic-cloud.com:9400",
        },
      },
    },
  },
});
console.log(response);

const response1 = await client.cluster.putSettings({
  persistent: {
    cluster: {
      remote: {
        clusterA: {
          mode: "proxy",
          skip_unavailable: true,
          server_name: "clustera.es.region-a.gcp.elastic-cloud.com",
          proxy_socket_connections: 18,
          proxy_address: "clustera.es.region-a.gcp.elastic-cloud.com:9400",
        },
      },
    },
  },
});
console.log(response1);

### On cluster A ###
PUT _cluster/settings
{
  "persistent": {
    "cluster": {
      "remote": {
        "clusterB": {
          "mode": "proxy",
          "skip_unavailable": true,
          "server_name": "clusterb.es.region-b.gcp.elastic-cloud.com",
          "proxy_socket_connections": 18,
          "proxy_address": "clusterb.es.region-b.gcp.elastic-cloud.com:9400"
        }
      }
    }
  }
}
### On cluster B ###
PUT _cluster/settings
{
  "persistent": {
    "cluster": {
      "remote": {
        "clusterA": {
          "mode": "proxy",
          "skip_unavailable": true,
          "server_name": "clustera.es.region-a.gcp.elastic-cloud.com",
          "proxy_socket_connections": 18,
          "proxy_address": "clustera.es.region-a.gcp.elastic-cloud.com:9400"
        }
      }
    }
  }
}

Set up bi-directional cross-cluster replication.

resp = client.ccr.put_auto_follow_pattern(
    name="logs-generic-default",
    remote_cluster="clusterB",
    leader_index_patterns=[
        ".ds-logs-generic-default-20*"
    ],
    leader_index_exclusion_patterns="*-replicated_from_clustera",
    follow_index_pattern="{{leader_index}}-replicated_from_clusterb",
)
print(resp)

resp1 = client.ccr.put_auto_follow_pattern(
    name="logs-generic-default",
    remote_cluster="clusterA",
    leader_index_patterns=[
        ".ds-logs-generic-default-20*"
    ],
    leader_index_exclusion_patterns="*-replicated_from_clusterb",
    follow_index_pattern="{{leader_index}}-replicated_from_clustera",
)
print(resp1)

const response = await client.ccr.putAutoFollowPattern({
  name: "logs-generic-default",
  remote_cluster: "clusterB",
  leader_index_patterns: [".ds-logs-generic-default-20*"],
  leader_index_exclusion_patterns: "*-replicated_from_clustera",
  follow_index_pattern: "{{leader_index}}-replicated_from_clusterb",
});
console.log(response);

const response1 = await client.ccr.putAutoFollowPattern({
  name: "logs-generic-default",
  remote_cluster: "clusterA",
  leader_index_patterns: [".ds-logs-generic-default-20*"],
  leader_index_exclusion_patterns: "*-replicated_from_clusterb",
  follow_index_pattern: "{{leader_index}}-replicated_from_clustera",
});
console.log(response1);

### On cluster A ###
PUT /_ccr/auto_follow/logs-generic-default
{
  "remote_cluster": "clusterB",
  "leader_index_patterns": [
    ".ds-logs-generic-default-20*"
  ],
  "leader_index_exclusion_patterns":"*-replicated_from_clustera",
  "follow_index_pattern": "{{leader_index}}-replicated_from_clusterb"
}

### On cluster B ###
PUT /_ccr/auto_follow/logs-generic-default
{
  "remote_cluster": "clusterA",
  "leader_index_patterns": [
    ".ds-logs-generic-default-20*"
  ],
  "leader_index_exclusion_patterns":"*-replicated_from_clusterb",
  "follow_index_pattern": "{{leader_index}}-replicated_from_clustera"
}

Existing data on the cluster will not be replicated by _ccr/auto_follow even though the patterns may match. This function will only replicate newly created backing indices (as part of the data stream).

Use leader_index_exclusion_patterns to avoid recursion.

follow_index_pattern allows lowercase characters only.

This step cannot be executed via the Kibana UI due to the lack of an exclusion pattern in the UI. Use the API in this step.

Set up the Logstash configuration file.

This example uses the input generator to demonstrate the document count in the clusters. Reconfigure this section to suit your own use case.
```
### On Logstash server ###
### This is a logstash config file ###
input {
  generator{
    message => 'Hello World'
    count => 100
  }
}
output {
  elasticsearch {
    hosts => ["https://clustera.es.region-a.gcp.elastic-cloud.com:9243","https://clusterb.es.region-b.gcp.elastic-cloud.com:9243"]
    user => "logstash-user"
    password => "same_password_for_both_clusters"
  }
}
```
The key point is that when cluster A is down, all traffic will be automatically redirected to cluster B. Once cluster A comes back, traffic is automatically redirected back to cluster A again. This is achieved by the option hosts where multiple ES cluster endpoints are specified in the array [clusterA, clusterB].

Set up the same password for the same user on both clusters to use this load-balancing feature.

Start Logstash with the earlier configuration file.

### On Logstash server ###
bin/logstash -f multiple_hosts.conf

Observe document counts in data streams.

The setup creates a data stream named logs-generic-default on each of the clusters. Logstash will write 50% of the documents to cluster A and 50% of the documents to cluster B when both clusters are up.

Bi-directional cross-cluster replication will create one more data stream on each of the clusters with the -replication_from_cluster{a|b} suffix. At the end of this step:
- data streams on cluster A contain:
  - 50 documents in logs-generic-default-replicated_from_clusterb
  - 50 documents in logs-generic-default
- data streams on cluster B contain:
  - 50 documents in logs-generic-default-replicated_from_clustera
  - 50 documents in logs-generic-default

Queries should be set up to search across both data streams. A query on logs*, on either of the clusters, returns 100 hits in total.

resp = client.search(
    index="logs*",
    size="0",
)
print(resp)

response = client.search(
  index: 'logs*',
  size: 0
)
puts response

const response = await client.search({
  index: "logs*",
  size: 0,
});
console.log(response);

GET logs*/_search?size=0

« Tutorial: Disaster recovery based on bi-directional cross-cluster replication Failover when clusterA is down »