Failover when clusterA is down

edit
  1. Promote the follower indices in clusterB into regular indices so that they accept writes. This can be achieved by:

    • First, pause indexing following for the follower index.
    • Next, close the follower index.
    • Unfollow the leader index.
    • Finally, open the follower index (which at this point is a regular index).
    resp = client.ccr.pause_follow(
        index="kibana_sample_data_ecommerce2",
    )
    print(resp)
    
    resp1 = client.indices.close(
        index="kibana_sample_data_ecommerce2",
    )
    print(resp1)
    
    resp2 = client.ccr.unfollow(
        index="kibana_sample_data_ecommerce2",
    )
    print(resp2)
    
    resp3 = client.indices.open(
        index="kibana_sample_data_ecommerce2",
    )
    print(resp3)
    response = client.ccr.pause_follow(
      index: 'kibana_sample_data_ecommerce2'
    )
    puts response
    
    response = client.indices.close(
      index: 'kibana_sample_data_ecommerce2'
    )
    puts response
    
    response = client.ccr.unfollow(
      index: 'kibana_sample_data_ecommerce2'
    )
    puts response
    
    response = client.indices.open(
      index: 'kibana_sample_data_ecommerce2'
    )
    puts response
    const response = await client.ccr.pauseFollow({
      index: "kibana_sample_data_ecommerce2",
    });
    console.log(response);
    
    const response1 = await client.indices.close({
      index: "kibana_sample_data_ecommerce2",
    });
    console.log(response1);
    
    const response2 = await client.ccr.unfollow({
      index: "kibana_sample_data_ecommerce2",
    });
    console.log(response2);
    
    const response3 = await client.indices.open({
      index: "kibana_sample_data_ecommerce2",
    });
    console.log(response3);
    ### On clusterB ###
    POST /kibana_sample_data_ecommerce2/_ccr/pause_follow
    POST /kibana_sample_data_ecommerce2/_close
    POST /kibana_sample_data_ecommerce2/_ccr/unfollow
    POST /kibana_sample_data_ecommerce2/_open
  2. On the client side (Logstash, Beats, Elastic Agent), manually re-enable ingestion of kibana_sample_data_ecommerce2 and redirect traffic to the clusterB. You should also redirect all search traffic to the clusterB cluster during this time. You can simulate this by ingesting documents into this index. You should notice this index is now writable.

    resp = client.index(
        index="kibana_sample_data_ecommerce2",
        document={
            "user": "kimchy"
        },
    )
    print(resp)
    response = client.index(
      index: 'kibana_sample_data_ecommerce2',
      body: {
        user: 'kimchy'
      }
    )
    puts response
    const response = await client.index({
      index: "kibana_sample_data_ecommerce2",
      document: {
        user: "kimchy",
      },
    });
    console.log(response);
    ### On clusterB ###
    POST kibana_sample_data_ecommerce2/_doc/
    {
      "user": "kimchy"
    }